14035 lines
493 KiB
JSON
14035 lines
493 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.6670224119530416,
|
|
"eval_steps": 500,
|
|
"global_step": 5000,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 543.9,
|
|
"completions/max_terminated_length": 543.6,
|
|
"completions/mean_length": 236.15,
|
|
"completions/mean_terminated_length": 235.96428604125975,
|
|
"completions/min_length": 104.4,
|
|
"completions/min_terminated_length": 104.4,
|
|
"entropy": 0.33476574160158634,
|
|
"epoch": 0.0013340448239060833,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.36328125,
|
|
"learning_rate": 2.4e-08,
|
|
"loss": 0.03974533379077912,
|
|
"num_tokens": 47060.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.22051151394844054,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.2205115258693695,
|
|
"step": 10,
|
|
"step_time": 12.462517058104277,
|
|
"zero_adv_skip_rate": 0.5
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 819.7,
|
|
"completions/max_terminated_length": 407.5,
|
|
"completions/mean_length": 273.0625,
|
|
"completions/mean_terminated_length": 203.68036270141602,
|
|
"completions/min_length": 83.6,
|
|
"completions/min_terminated_length": 83.6,
|
|
"entropy": 0.355833588540554,
|
|
"epoch": 0.0026680896478121665,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5.0666666666666664e-08,
|
|
"loss": 0.10912301540374755,
|
|
"num_tokens": 99089.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 20,
|
|
"step_time": 18.261661757342516,
|
|
"zero_adv_skip_rate": 0.5
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 243.6,
|
|
"completions/max_terminated_length": 211.3,
|
|
"completions/mean_length": 146.5,
|
|
"completions/mean_terminated_length": 141.15893020629883,
|
|
"completions/min_length": 71.9,
|
|
"completions/min_terminated_length": 71.9,
|
|
"entropy": 0.3921903227455914,
|
|
"epoch": 0.00400213447171825,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 1.1640625,
|
|
"learning_rate": 7.733333333333334e-08,
|
|
"loss": -0.059109818935394284,
|
|
"num_tokens": 142465.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.36653792560100557,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.36653794944286344,
|
|
"step": 30,
|
|
"step_time": 6.762725382577628,
|
|
"zero_adv_skip_rate": 0.4
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 669.5,
|
|
"completions/max_terminated_length": 304.2,
|
|
"completions/mean_length": 215.775,
|
|
"completions/mean_terminated_length": 167.61964492797853,
|
|
"completions/min_length": 70.0,
|
|
"completions/min_terminated_length": 70.0,
|
|
"entropy": 0.3185795673634857,
|
|
"epoch": 0.005336179295624333,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.46875,
|
|
"learning_rate": 1.0399999999999999e-07,
|
|
"loss": 0.10917320251464843,
|
|
"num_tokens": 185975.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.19231741726398469,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.19231742918491362,
|
|
"step": 40,
|
|
"step_time": 14.277801349479706,
|
|
"zero_adv_skip_rate": 0.45
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 695.1,
|
|
"completions/max_terminated_length": 463.1,
|
|
"completions/mean_length": 280.275,
|
|
"completions/mean_terminated_length": 221.4857147216797,
|
|
"completions/min_length": 76.9,
|
|
"completions/min_terminated_length": 76.9,
|
|
"entropy": 0.5089598641498014,
|
|
"epoch": 0.006670224119530417,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1.3066666666666665e-07,
|
|
"loss": 0.04101671278476715,
|
|
"num_tokens": 240349.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.3501461952924728,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.35014620125293733,
|
|
"step": 50,
|
|
"step_time": 15.486472485214472,
|
|
"zero_adv_skip_rate": 0.4
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 524.2,
|
|
"completions/max_terminated_length": 338.6,
|
|
"completions/mean_length": 192.4375,
|
|
"completions/mean_terminated_length": 168.37678604125978,
|
|
"completions/min_length": 78.4,
|
|
"completions/min_terminated_length": 78.4,
|
|
"entropy": 0.43015456836049754,
|
|
"epoch": 0.0080042689434365,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.39453125,
|
|
"learning_rate": 1.573333333333333e-07,
|
|
"loss": 0.10799503326416016,
|
|
"num_tokens": 287448.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.2575641840696335,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.25756419003009795,
|
|
"step": 60,
|
|
"step_time": 11.948348375223578,
|
|
"zero_adv_skip_rate": 0.4
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 608.5,
|
|
"completions/max_terminated_length": 500.8,
|
|
"completions/mean_length": 241.3375,
|
|
"completions/mean_terminated_length": 220.94464569091798,
|
|
"completions/min_length": 79.0,
|
|
"completions/min_terminated_length": 79.0,
|
|
"entropy": 0.4609580057274018,
|
|
"epoch": 0.009338313767342583,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.3828125,
|
|
"learning_rate": 1.8399999999999998e-07,
|
|
"loss": 0.055780434608459474,
|
|
"num_tokens": 335923.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.336654344201088,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.3366543620824814,
|
|
"step": 70,
|
|
"step_time": 13.85979171069339,
|
|
"zero_adv_skip_rate": 0.3857
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 258.7,
|
|
"completions/max_terminated_length": 254.5,
|
|
"completions/mean_length": 141.35,
|
|
"completions/mean_terminated_length": 140.18571548461915,
|
|
"completions/min_length": 69.6,
|
|
"completions/min_terminated_length": 69.6,
|
|
"entropy": 0.3707403258553573,
|
|
"epoch": 0.010672358591248666,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.67578125,
|
|
"learning_rate": 2.1066666666666665e-07,
|
|
"loss": -0.028505098819732667,
|
|
"num_tokens": 373783.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.3294931024312973,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.32949312031269073,
|
|
"step": 80,
|
|
"step_time": 6.787669274304062,
|
|
"zero_adv_skip_rate": 0.375
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 349.1,
|
|
"completions/max_terminated_length": 324.0,
|
|
"completions/mean_length": 156.575,
|
|
"completions/mean_terminated_length": 151.4357147216797,
|
|
"completions/min_length": 78.3,
|
|
"completions/min_terminated_length": 78.3,
|
|
"entropy": 0.37145454303494524,
|
|
"epoch": 0.01200640341515475,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.51953125,
|
|
"learning_rate": 2.3733333333333334e-07,
|
|
"loss": 0.06838004589080811,
|
|
"num_tokens": 415333.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.35305392146110537,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.35305394530296325,
|
|
"step": 90,
|
|
"step_time": 8.615415358450264,
|
|
"zero_adv_skip_rate": 0.3667
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 712.0,
|
|
"completions/max_terminated_length": 546.7,
|
|
"completions/mean_length": 231.65,
|
|
"completions/mean_terminated_length": 209.44642868041993,
|
|
"completions/min_length": 89.2,
|
|
"completions/min_terminated_length": 89.2,
|
|
"entropy": 0.3282428504899144,
|
|
"epoch": 0.013340448239060833,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.64e-07,
|
|
"loss": 0.03210486769676209,
|
|
"num_tokens": 462617.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.28112500309944155,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.28112501502037046,
|
|
"step": 100,
|
|
"step_time": 15.820801395736634,
|
|
"zero_adv_skip_rate": 0.37
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 665.3,
|
|
"completions/max_terminated_length": 665.3,
|
|
"completions/mean_length": 282.9,
|
|
"completions/mean_terminated_length": 282.9,
|
|
"completions/min_length": 108.3,
|
|
"completions/min_terminated_length": 108.3,
|
|
"entropy": 0.49002441577613354,
|
|
"epoch": 0.014674493062966915,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.906666666666667e-07,
|
|
"loss": 0.011400097608566284,
|
|
"num_tokens": 518433.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.1796922653913498,
|
|
"rewards/correctness_reward/mean": 0.125,
|
|
"rewards/correctness_reward/std": 0.17969227135181426,
|
|
"step": 110,
|
|
"step_time": 14.495567450858653,
|
|
"zero_adv_skip_rate": 0.3909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 362.0,
|
|
"completions/max_terminated_length": 361.8,
|
|
"completions/mean_length": 190.95,
|
|
"completions/mean_terminated_length": 190.7732147216797,
|
|
"completions/min_length": 81.3,
|
|
"completions/min_terminated_length": 81.3,
|
|
"entropy": 0.49063170111427706,
|
|
"epoch": 0.016008537886873,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.173333333333333e-07,
|
|
"loss": 0.06074678897857666,
|
|
"num_tokens": 561733.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.21213203072547912,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.21213203072547912,
|
|
"step": 120,
|
|
"step_time": 8.546123830880969,
|
|
"zero_adv_skip_rate": 0.3917
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 964.6,
|
|
"completions/max_terminated_length": 630.9,
|
|
"completions/mean_length": 295.825,
|
|
"completions/mean_terminated_length": 234.458935546875,
|
|
"completions/min_length": 98.3,
|
|
"completions/min_terminated_length": 98.3,
|
|
"entropy": 0.36501956451684237,
|
|
"epoch": 0.017342582710779084,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.4399999999999996e-07,
|
|
"loss": 0.1221515417098999,
|
|
"num_tokens": 614903.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.1414213538169861,
|
|
"rewards/correctness_reward/mean": 0.2,
|
|
"rewards/correctness_reward/std": 0.1414213538169861,
|
|
"step": 130,
|
|
"step_time": 20.085508758109064,
|
|
"zero_adv_skip_rate": 0.4077
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 855.5,
|
|
"completions/max_terminated_length": 357.2,
|
|
"completions/mean_length": 263.9125,
|
|
"completions/mean_terminated_length": 194.28928756713867,
|
|
"completions/min_length": 71.4,
|
|
"completions/min_terminated_length": 71.4,
|
|
"entropy": 0.5393201597034931,
|
|
"epoch": 0.018676627534685165,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.7066666666666665e-07,
|
|
"loss": -0.03463023900985718,
|
|
"num_tokens": 664968.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.19864802658557892,
|
|
"rewards/correctness_reward/mean": 0.0875,
|
|
"rewards/correctness_reward/std": 0.19864802658557892,
|
|
"step": 140,
|
|
"step_time": 18.248422381468117,
|
|
"zero_adv_skip_rate": 0.4143
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 281.4,
|
|
"completions/max_terminated_length": 277.8,
|
|
"completions/mean_length": 127.4625,
|
|
"completions/mean_terminated_length": 121.63571548461914,
|
|
"completions/min_length": 65.2,
|
|
"completions/min_terminated_length": 65.2,
|
|
"entropy": 0.5574440363794565,
|
|
"epoch": 0.020010672358591247,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.973333333333333e-07,
|
|
"loss": 0.04595287442207337,
|
|
"num_tokens": 703581.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.23314451277256013,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.2331445187330246,
|
|
"step": 150,
|
|
"step_time": 6.940428514778614,
|
|
"zero_adv_skip_rate": 0.42
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 620.6,
|
|
"completions/max_terminated_length": 219.0,
|
|
"completions/mean_length": 181.15,
|
|
"completions/mean_terminated_length": 128.56071586608886,
|
|
"completions/min_length": 70.3,
|
|
"completions/min_terminated_length": 70.3,
|
|
"entropy": 0.4845874408259988,
|
|
"epoch": 0.021344717182497332,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.24e-07,
|
|
"loss": 0.25077483654022215,
|
|
"num_tokens": 749417.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.22220884561538695,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.22220885157585143,
|
|
"step": 160,
|
|
"step_time": 13.599445314519107,
|
|
"zero_adv_skip_rate": 0.425
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 553.5,
|
|
"completions/max_terminated_length": 369.0,
|
|
"completions/mean_length": 217.775,
|
|
"completions/mean_terminated_length": 192.09821548461915,
|
|
"completions/min_length": 90.7,
|
|
"completions/min_terminated_length": 90.7,
|
|
"entropy": 0.44597798120230436,
|
|
"epoch": 0.022678762006403414,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.84375,
|
|
"learning_rate": 4.506666666666666e-07,
|
|
"loss": 0.010445863008499146,
|
|
"num_tokens": 794927.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.24662851691246032,
|
|
"rewards/correctness_reward/mean": 0.1625,
|
|
"rewards/correctness_reward/std": 0.2466285228729248,
|
|
"step": 170,
|
|
"step_time": 12.486412231158464,
|
|
"zero_adv_skip_rate": 0.4235
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 685.7,
|
|
"completions/max_terminated_length": 389.5,
|
|
"completions/mean_length": 302.9,
|
|
"completions/mean_terminated_length": 216.44166870117186,
|
|
"completions/min_length": 103.4,
|
|
"completions/min_terminated_length": 103.4,
|
|
"entropy": 0.3101295493543148,
|
|
"epoch": 0.0240128068303095,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.773333333333333e-07,
|
|
"loss": 0.033603540062904357,
|
|
"num_tokens": 857519.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.0816463440656662,
|
|
"rewards/correctness_reward/mean": 0.2125,
|
|
"rewards/correctness_reward/std": 0.0816463440656662,
|
|
"step": 180,
|
|
"step_time": 14.65678074322641,
|
|
"zero_adv_skip_rate": 0.4444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 356.3,
|
|
"completions/max_terminated_length": 356.3,
|
|
"completions/mean_length": 157.5375,
|
|
"completions/mean_terminated_length": 157.5375,
|
|
"completions/min_length": 77.3,
|
|
"completions/min_terminated_length": 77.3,
|
|
"entropy": 0.4195747395977378,
|
|
"epoch": 0.02534685165421558,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5.04e-07,
|
|
"loss": 0.08028936982154847,
|
|
"num_tokens": 898538.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.23314451277256013,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.2331445187330246,
|
|
"step": 190,
|
|
"step_time": 8.40141998846084,
|
|
"zero_adv_skip_rate": 0.4474
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 534.7,
|
|
"completions/max_terminated_length": 389.8,
|
|
"completions/mean_length": 219.7125,
|
|
"completions/mean_terminated_length": 197.04107284545898,
|
|
"completions/min_length": 71.7,
|
|
"completions/min_terminated_length": 71.7,
|
|
"entropy": 0.4439691748470068,
|
|
"epoch": 0.026680896478121666,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.66015625,
|
|
"learning_rate": 5.306666666666665e-07,
|
|
"loss": -0.035671621561050415,
|
|
"num_tokens": 949787.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.24200308620929717,
|
|
"rewards/correctness_reward/mean": 0.2375,
|
|
"rewards/correctness_reward/std": 0.24200308620929717,
|
|
"step": 200,
|
|
"step_time": 11.756518095824868,
|
|
"zero_adv_skip_rate": 0.45
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 432.2,
|
|
"completions/max_terminated_length": 345.2,
|
|
"completions/mean_length": 192.575,
|
|
"completions/mean_terminated_length": 170.0875030517578,
|
|
"completions/min_length": 75.3,
|
|
"completions/min_terminated_length": 75.3,
|
|
"entropy": 0.3814262927820285,
|
|
"epoch": 0.028014941302027748,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.78125,
|
|
"learning_rate": 5.573333333333333e-07,
|
|
"loss": -0.023694081604480742,
|
|
"num_tokens": 994073.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.2832020968198776,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.28320211470127105,
|
|
"step": 210,
|
|
"step_time": 9.918385525420309,
|
|
"zero_adv_skip_rate": 0.4476
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 481.1,
|
|
"completions/max_terminated_length": 320.3,
|
|
"completions/mean_length": 185.4875,
|
|
"completions/mean_terminated_length": 162.13035736083984,
|
|
"completions/min_length": 77.1,
|
|
"completions/min_terminated_length": 77.1,
|
|
"entropy": 0.4142140754395061,
|
|
"epoch": 0.02934898612593383,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.498046875,
|
|
"learning_rate": 5.839999999999999e-07,
|
|
"loss": -0.02750779390335083,
|
|
"num_tokens": 1037416.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.41999801993370056,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.4199980318546295,
|
|
"step": 220,
|
|
"step_time": 11.41794089321047,
|
|
"zero_adv_skip_rate": 0.4318
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 468.5,
|
|
"completions/max_terminated_length": 300.0,
|
|
"completions/mean_length": 158.05,
|
|
"completions/mean_terminated_length": 132.67678756713866,
|
|
"completions/min_length": 57.1,
|
|
"completions/min_terminated_length": 57.1,
|
|
"entropy": 0.4031065130340202,
|
|
"epoch": 0.030683030949839915,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 6.106666666666666e-07,
|
|
"loss": -0.052559852600097656,
|
|
"num_tokens": 1077396.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.3257265239953995,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.32572652995586393,
|
|
"step": 230,
|
|
"step_time": 10.807588554359972,
|
|
"zero_adv_skip_rate": 0.4261
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 594.1,
|
|
"completions/max_terminated_length": 481.3,
|
|
"completions/mean_length": 222.4625,
|
|
"completions/mean_terminated_length": 202.2053581237793,
|
|
"completions/min_length": 76.5,
|
|
"completions/min_terminated_length": 76.5,
|
|
"entropy": 0.37266330467537045,
|
|
"epoch": 0.032017075773746,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 1.046875,
|
|
"learning_rate": 6.373333333333333e-07,
|
|
"loss": -0.002203541994094849,
|
|
"num_tokens": 1121353.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.28112500309944155,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.28112501502037046,
|
|
"step": 240,
|
|
"step_time": 13.164311448484659,
|
|
"zero_adv_skip_rate": 0.425
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 966.1,
|
|
"completions/max_terminated_length": 647.0,
|
|
"completions/mean_length": 351.925,
|
|
"completions/mean_terminated_length": 268.89167098999025,
|
|
"completions/min_length": 111.8,
|
|
"completions/min_terminated_length": 111.8,
|
|
"entropy": 0.3341396586038172,
|
|
"epoch": 0.03335112059765208,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 6.64e-07,
|
|
"loss": 0.00015666782855987548,
|
|
"num_tokens": 1176587.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.19231741726398469,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.19231742918491362,
|
|
"step": 250,
|
|
"step_time": 20.236098586022855,
|
|
"zero_adv_skip_rate": 0.432
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 366.4,
|
|
"completions/max_terminated_length": 365.6,
|
|
"completions/mean_length": 182.5,
|
|
"completions/mean_terminated_length": 182.0482147216797,
|
|
"completions/min_length": 61.5,
|
|
"completions/min_terminated_length": 61.5,
|
|
"entropy": 0.4536341023631394,
|
|
"epoch": 0.03468516542155817,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 1.6953125,
|
|
"learning_rate": 6.906666666666666e-07,
|
|
"loss": 0.02451373338699341,
|
|
"num_tokens": 1222667.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.38463483452796937,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.38463485836982725,
|
|
"step": 260,
|
|
"step_time": 8.996326134167612,
|
|
"zero_adv_skip_rate": 0.4231
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 609.9,
|
|
"completions/max_terminated_length": 413.1,
|
|
"completions/mean_length": 221.6625,
|
|
"completions/mean_terminated_length": 179.9125057220459,
|
|
"completions/min_length": 68.1,
|
|
"completions/min_terminated_length": 68.1,
|
|
"entropy": 0.3698773165898664,
|
|
"epoch": 0.03601921024546425,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 7.173333333333333e-07,
|
|
"loss": 0.004571209847927094,
|
|
"num_tokens": 1270136.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.336654344201088,
|
|
"rewards/correctness_reward/mean": 0.45,
|
|
"rewards/correctness_reward/std": 0.3366543620824814,
|
|
"step": 270,
|
|
"step_time": 13.866699990443886,
|
|
"zero_adv_skip_rate": 0.4185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 466.9,
|
|
"completions/max_terminated_length": 297.9,
|
|
"completions/mean_length": 176.6,
|
|
"completions/mean_terminated_length": 154.08214569091797,
|
|
"completions/min_length": 77.9,
|
|
"completions/min_terminated_length": 77.9,
|
|
"entropy": 0.5338103393358844,
|
|
"epoch": 0.03735325506937033,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 1.0078125,
|
|
"learning_rate": 7.44e-07,
|
|
"loss": 0.02621854543685913,
|
|
"num_tokens": 1315368.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.31479085683822633,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.3147908627986908,
|
|
"step": 280,
|
|
"step_time": 10.979100323375315,
|
|
"zero_adv_skip_rate": 0.4143
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 418.9,
|
|
"completions/max_terminated_length": 413.7,
|
|
"completions/mean_length": 185.3625,
|
|
"completions/mean_terminated_length": 184.25178604125978,
|
|
"completions/min_length": 67.3,
|
|
"completions/min_terminated_length": 67.3,
|
|
"entropy": 0.46728595186557087,
|
|
"epoch": 0.03868729989327641,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 7.706666666666667e-07,
|
|
"loss": 0.06420769691467285,
|
|
"num_tokens": 1359765.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.29122219085693357,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.29122220277786254,
|
|
"step": 290,
|
|
"step_time": 10.011226489208639,
|
|
"zero_adv_skip_rate": 0.4103
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 1057.9,
|
|
"completions/max_terminated_length": 878.0,
|
|
"completions/mean_length": 305.575,
|
|
"completions/mean_terminated_length": 281.70000038146975,
|
|
"completions/min_length": 97.8,
|
|
"completions/min_terminated_length": 97.8,
|
|
"entropy": 0.38504726393148303,
|
|
"epoch": 0.040021344717182494,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 7.973333333333333e-07,
|
|
"loss": -0.0658410668373108,
|
|
"num_tokens": 1412659.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.16875659823417663,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.1687566041946411,
|
|
"step": 300,
|
|
"step_time": 22.131056026089936,
|
|
"zero_adv_skip_rate": 0.4167
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 454.3,
|
|
"completions/max_terminated_length": 444.9,
|
|
"completions/mean_length": 151.35,
|
|
"completions/mean_terminated_length": 148.50357246398926,
|
|
"completions/min_length": 63.9,
|
|
"completions/min_terminated_length": 63.9,
|
|
"entropy": 0.4179494706913829,
|
|
"epoch": 0.04135538954108858,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 8.24e-07,
|
|
"loss": 0.1620602250099182,
|
|
"num_tokens": 1453375.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.08711025416851044,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.08711026012897491,
|
|
"step": 310,
|
|
"step_time": 10.326902017183603,
|
|
"zero_adv_skip_rate": 0.429
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 601.1,
|
|
"completions/max_terminated_length": 422.7,
|
|
"completions/mean_length": 234.7,
|
|
"completions/mean_terminated_length": 208.71250228881837,
|
|
"completions/min_length": 84.5,
|
|
"completions/min_terminated_length": 84.5,
|
|
"entropy": 0.5138832532490293,
|
|
"epoch": 0.042689434364994665,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.60546875,
|
|
"learning_rate": 8.506666666666667e-07,
|
|
"loss": -0.0625089943408966,
|
|
"num_tokens": 1502127.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.2558668524026871,
|
|
"rewards/correctness_reward/mean": 0.2375,
|
|
"rewards/correctness_reward/std": 0.25586686432361605,
|
|
"step": 320,
|
|
"step_time": 13.70193090569228,
|
|
"zero_adv_skip_rate": 0.4281
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 715.2,
|
|
"completions/max_terminated_length": 701.8,
|
|
"completions/mean_length": 291.2375,
|
|
"completions/mean_terminated_length": 273.86607666015624,
|
|
"completions/min_length": 91.4,
|
|
"completions/min_terminated_length": 91.4,
|
|
"entropy": 0.39913020038511604,
|
|
"epoch": 0.044023479188900747,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.83203125,
|
|
"learning_rate": 8.773333333333332e-07,
|
|
"loss": -0.0006826147437095642,
|
|
"num_tokens": 1552826.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.3665457725524902,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.3665457844734192,
|
|
"step": 330,
|
|
"step_time": 16.070000430662184,
|
|
"zero_adv_skip_rate": 0.4212
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 736.8,
|
|
"completions/max_terminated_length": 576.4,
|
|
"completions/mean_length": 255.1125,
|
|
"completions/mean_terminated_length": 231.24285736083985,
|
|
"completions/min_length": 76.8,
|
|
"completions/min_terminated_length": 76.8,
|
|
"entropy": 0.47501376777654514,
|
|
"epoch": 0.04535752401280683,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.44921875,
|
|
"learning_rate": 9.039999999999999e-07,
|
|
"loss": 0.06209576725959778,
|
|
"num_tokens": 1616443.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.17422050833702088,
|
|
"rewards/correctness_reward/mean": 0.1,
|
|
"rewards/correctness_reward/std": 0.17422052025794982,
|
|
"step": 340,
|
|
"step_time": 16.075904679577796,
|
|
"zero_adv_skip_rate": 0.4265
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 709.5,
|
|
"completions/max_terminated_length": 512.6,
|
|
"completions/mean_length": 214.875,
|
|
"completions/mean_terminated_length": 188.73750114440918,
|
|
"completions/min_length": 65.1,
|
|
"completions/min_terminated_length": 65.1,
|
|
"entropy": 0.35718262302024023,
|
|
"epoch": 0.04669156883671292,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.68359375,
|
|
"learning_rate": 9.306666666666666e-07,
|
|
"loss": 0.060851722955703735,
|
|
"num_tokens": 1663153.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.31479085683822633,
|
|
"rewards/correctness_reward/mean": 0.3375,
|
|
"rewards/correctness_reward/std": 0.3147908627986908,
|
|
"step": 350,
|
|
"step_time": 15.802935469150544,
|
|
"zero_adv_skip_rate": 0.4229
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 543.5,
|
|
"completions/max_terminated_length": 518.3,
|
|
"completions/mean_length": 277.6375,
|
|
"completions/mean_terminated_length": 247.9761947631836,
|
|
"completions/min_length": 90.7,
|
|
"completions/min_terminated_length": 90.7,
|
|
"entropy": 0.37842598925344645,
|
|
"epoch": 0.048025613660619,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.71484375,
|
|
"learning_rate": 9.573333333333333e-07,
|
|
"loss": -0.08970316648483276,
|
|
"num_tokens": 1713956.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.4372564494609833,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.4372564613819122,
|
|
"step": 360,
|
|
"step_time": 12.859038892295212,
|
|
"zero_adv_skip_rate": 0.4111
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 540.2,
|
|
"completions/max_terminated_length": 341.8,
|
|
"completions/mean_length": 192.4875,
|
|
"completions/mean_terminated_length": 166.52142944335938,
|
|
"completions/min_length": 78.7,
|
|
"completions/min_terminated_length": 78.7,
|
|
"entropy": 0.41244630174090463,
|
|
"epoch": 0.04935965848452508,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 1.0,
|
|
"learning_rate": 9.84e-07,
|
|
"loss": 0.03358766138553619,
|
|
"num_tokens": 1760243.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.2756610929965973,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.27566109895706176,
|
|
"step": 370,
|
|
"step_time": 12.296358043793589,
|
|
"zero_adv_skip_rate": 0.4108
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 1011.0,
|
|
"completions/max_terminated_length": 591.3,
|
|
"completions/mean_length": 317.9875,
|
|
"completions/mean_terminated_length": 246.46607360839843,
|
|
"completions/min_length": 82.9,
|
|
"completions/min_terminated_length": 82.9,
|
|
"entropy": 0.5527587074320763,
|
|
"epoch": 0.05069370330843116,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.00881413072347641,
|
|
"num_tokens": 1814970.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.16875659823417663,
|
|
"rewards/correctness_reward/mean": 0.1125,
|
|
"rewards/correctness_reward/std": 0.1687566041946411,
|
|
"step": 380,
|
|
"step_time": 21.43901033885777,
|
|
"zero_adv_skip_rate": 0.4158
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 350.2,
|
|
"completions/max_terminated_length": 292.9,
|
|
"completions/mean_length": 183.3625,
|
|
"completions/mean_terminated_length": 171.9660758972168,
|
|
"completions/min_length": 81.3,
|
|
"completions/min_terminated_length": 81.3,
|
|
"entropy": 0.40587806492112577,
|
|
"epoch": 0.052027748132337244,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.01812410056591034,
|
|
"num_tokens": 1861375.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.15782093107700348,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.15782093703746797,
|
|
"step": 390,
|
|
"step_time": 8.115243971254676,
|
|
"zero_adv_skip_rate": 0.4205
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 501.8,
|
|
"completions/max_terminated_length": 315.2,
|
|
"completions/mean_length": 192.4625,
|
|
"completions/mean_terminated_length": 143.14881134033203,
|
|
"completions/min_length": 66.6,
|
|
"completions/min_terminated_length": 66.6,
|
|
"entropy": 0.39273061347194016,
|
|
"epoch": 0.05336179295624333,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.010032571852207184,
|
|
"num_tokens": 1903092.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.332049286365509,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.3320492923259735,
|
|
"step": 400,
|
|
"step_time": 11.721796408202499,
|
|
"zero_adv_skip_rate": 0.415
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 695.2,
|
|
"completions/max_terminated_length": 541.3,
|
|
"completions/mean_length": 247.625,
|
|
"completions/mean_terminated_length": 224.30535736083985,
|
|
"completions/min_length": 86.3,
|
|
"completions/min_terminated_length": 86.3,
|
|
"entropy": 0.5935859555999438,
|
|
"epoch": 0.054695837780149414,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.011779013276100158,
|
|
"num_tokens": 1952862.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.13886516988277436,
|
|
"rewards/correctness_reward/mean": 0.1375,
|
|
"rewards/correctness_reward/std": 0.1388651818037033,
|
|
"step": 410,
|
|
"step_time": 14.834286655671894,
|
|
"zero_adv_skip_rate": 0.422
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 760.3,
|
|
"completions/max_terminated_length": 523.0,
|
|
"completions/mean_length": 300.2875,
|
|
"completions/mean_terminated_length": 253.42857666015624,
|
|
"completions/min_length": 116.0,
|
|
"completions/min_terminated_length": 116.0,
|
|
"entropy": 0.3548567413041989,
|
|
"epoch": 0.056029882604055496,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.013163346052169799,
|
|
"num_tokens": 2008397.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.15149816870689392,
|
|
"rewards/correctness_reward/mean": 0.1375,
|
|
"rewards/correctness_reward/std": 0.1514981746673584,
|
|
"step": 420,
|
|
"step_time": 16.27429291754961,
|
|
"zero_adv_skip_rate": 0.4286
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 623.4,
|
|
"completions/max_terminated_length": 616.3,
|
|
"completions/mean_length": 197.825,
|
|
"completions/mean_terminated_length": 196.225,
|
|
"completions/min_length": 69.5,
|
|
"completions/min_terminated_length": 69.5,
|
|
"entropy": 0.4613298310173882,
|
|
"epoch": 0.05736392742796158,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.408203125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.06693354845046998,
|
|
"num_tokens": 2055855.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.35100504755973816,
|
|
"rewards/correctness_reward/mean": 0.2,
|
|
"rewards/correctness_reward/std": 0.35100504755973816,
|
|
"step": 430,
|
|
"step_time": 14.261557662300765,
|
|
"zero_adv_skip_rate": 0.4209
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 703.5,
|
|
"completions/max_terminated_length": 568.4,
|
|
"completions/mean_length": 273.2,
|
|
"completions/mean_terminated_length": 244.36428680419922,
|
|
"completions/min_length": 93.3,
|
|
"completions/min_terminated_length": 93.3,
|
|
"entropy": 0.3966683746049447,
|
|
"epoch": 0.05869797225186766,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.3046875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.07529031634330749,
|
|
"num_tokens": 2106151.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.3110164314508438,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.3110164374113083,
|
|
"step": 440,
|
|
"step_time": 15.617533381376415,
|
|
"zero_adv_skip_rate": 0.4182
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 514.8,
|
|
"completions/max_terminated_length": 490.2,
|
|
"completions/mean_length": 223.85,
|
|
"completions/mean_terminated_length": 216.32678756713867,
|
|
"completions/min_length": 90.1,
|
|
"completions/min_terminated_length": 90.1,
|
|
"entropy": 0.33994191605597734,
|
|
"epoch": 0.06003201707577375,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0443997323513031,
|
|
"num_tokens": 2152587.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.23144718110561371,
|
|
"rewards/correctness_reward/mean": 0.2375,
|
|
"rewards/correctness_reward/std": 0.23144719302654265,
|
|
"step": 450,
|
|
"step_time": 11.667785560525953,
|
|
"zero_adv_skip_rate": 0.42
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 240.8,
|
|
"completions/max_terminated_length": 225.9,
|
|
"completions/mean_length": 128.9125,
|
|
"completions/mean_terminated_length": 125.66607246398925,
|
|
"completions/min_length": 70.4,
|
|
"completions/min_terminated_length": 70.4,
|
|
"entropy": 0.4598073702305555,
|
|
"epoch": 0.06136606189967983,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.02991713285446167,
|
|
"num_tokens": 2189044.0,
|
|
"reward": 0.5875,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.5875,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 460,
|
|
"step_time": 6.172788832895458,
|
|
"zero_adv_skip_rate": 0.4217
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 679.2,
|
|
"completions/max_terminated_length": 500.2,
|
|
"completions/mean_length": 234.4375,
|
|
"completions/mean_terminated_length": 210.22857398986815,
|
|
"completions/min_length": 83.2,
|
|
"completions/min_terminated_length": 83.2,
|
|
"entropy": 0.4345310034696013,
|
|
"epoch": 0.06270010672358592,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.30078125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.07599543929100036,
|
|
"num_tokens": 2236615.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.3265853762626648,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.3265853762626648,
|
|
"step": 470,
|
|
"step_time": 15.359032003302127,
|
|
"zero_adv_skip_rate": 0.417
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 659.6,
|
|
"completions/max_terminated_length": 551.5,
|
|
"completions/mean_length": 232.3125,
|
|
"completions/mean_terminated_length": 212.05000381469728,
|
|
"completions/min_length": 67.5,
|
|
"completions/min_terminated_length": 67.5,
|
|
"entropy": 0.541877678129822,
|
|
"epoch": 0.064034151547492,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 1.6796875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.11774296760559082,
|
|
"num_tokens": 2285320.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.16875659823417663,
|
|
"rewards/correctness_reward/mean": 0.3375,
|
|
"rewards/correctness_reward/std": 0.1687566041946411,
|
|
"step": 480,
|
|
"step_time": 14.460040126368403,
|
|
"zero_adv_skip_rate": 0.4208
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 391.2,
|
|
"completions/max_terminated_length": 233.5,
|
|
"completions/mean_length": 168.1,
|
|
"completions/mean_terminated_length": 143.55893173217774,
|
|
"completions/min_length": 69.1,
|
|
"completions/min_terminated_length": 69.1,
|
|
"entropy": 0.3079519834635513,
|
|
"epoch": 0.06536819637139808,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.035642051696777345,
|
|
"num_tokens": 2325288.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.3185652822256088,
|
|
"rewards/correctness_reward/mean": 0.5,
|
|
"rewards/correctness_reward/std": 0.3185652881860733,
|
|
"step": 490,
|
|
"step_time": 9.453113468363881,
|
|
"zero_adv_skip_rate": 0.4184
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 517.9,
|
|
"completions/max_terminated_length": 320.7,
|
|
"completions/mean_length": 164.3875,
|
|
"completions/mean_terminated_length": 138.57857246398925,
|
|
"completions/min_length": 55.1,
|
|
"completions/min_terminated_length": 55.1,
|
|
"entropy": 0.45513398642651737,
|
|
"epoch": 0.06670224119530416,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 1.0625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.18826091289520264,
|
|
"num_tokens": 2368967.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.38086825609207153,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.3808682680130005,
|
|
"step": 500,
|
|
"step_time": 12.191454120818525,
|
|
"zero_adv_skip_rate": 0.414
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 551.3,
|
|
"completions/max_terminated_length": 399.3,
|
|
"completions/mean_length": 212.175,
|
|
"completions/mean_terminated_length": 190.9839294433594,
|
|
"completions/min_length": 96.9,
|
|
"completions/min_terminated_length": 96.9,
|
|
"entropy": 0.48386048153042793,
|
|
"epoch": 0.06803628601921025,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 1.8203125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.06976993680000305,
|
|
"num_tokens": 2413013.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.3130935251712799,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.3130935370922089,
|
|
"step": 510,
|
|
"step_time": 12.906797604076564,
|
|
"zero_adv_skip_rate": 0.4118
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 673.0,
|
|
"completions/max_terminated_length": 326.2,
|
|
"completions/mean_length": 198.3625,
|
|
"completions/mean_terminated_length": 143.99107360839844,
|
|
"completions/min_length": 75.8,
|
|
"completions/min_terminated_length": 75.8,
|
|
"entropy": 0.38246993640703814,
|
|
"epoch": 0.06937033084311633,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.796875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1208719253540039,
|
|
"num_tokens": 2458314.0,
|
|
"reward": 0.5375,
|
|
"reward_std": 0.3021578580141068,
|
|
"rewards/correctness_reward/mean": 0.5375,
|
|
"rewards/correctness_reward/std": 0.3021578699350357,
|
|
"step": 520,
|
|
"step_time": 15.103981120698155,
|
|
"zero_adv_skip_rate": 0.4096
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 436.3,
|
|
"completions/max_terminated_length": 242.9,
|
|
"completions/mean_length": 164.2375,
|
|
"completions/mean_terminated_length": 139.0482151031494,
|
|
"completions/min_length": 72.5,
|
|
"completions/min_terminated_length": 72.5,
|
|
"entropy": 0.4249820187687874,
|
|
"epoch": 0.07070437566702241,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.345703125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.09194407463073731,
|
|
"num_tokens": 2499085.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.44017202258110044,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.4401720404624939,
|
|
"step": 530,
|
|
"step_time": 10.710739918518811,
|
|
"zero_adv_skip_rate": 0.4038
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 578.0,
|
|
"completions/max_terminated_length": 508.4,
|
|
"completions/mean_length": 252.6625,
|
|
"completions/mean_terminated_length": 236.33214416503907,
|
|
"completions/min_length": 105.8,
|
|
"completions/min_terminated_length": 105.8,
|
|
"entropy": 0.4418235430493951,
|
|
"epoch": 0.0720384204909285,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.3671875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.03516019582748413,
|
|
"num_tokens": 2548970.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.15235702097415924,
|
|
"rewards/correctness_reward/mean": 0.1375,
|
|
"rewards/correctness_reward/std": 0.15235702097415924,
|
|
"step": 540,
|
|
"step_time": 13.172187670413404,
|
|
"zero_adv_skip_rate": 0.4074
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 841.8,
|
|
"completions/max_terminated_length": 630.6,
|
|
"completions/mean_length": 304.1,
|
|
"completions/mean_terminated_length": 268.8517879486084,
|
|
"completions/min_length": 100.8,
|
|
"completions/min_terminated_length": 100.8,
|
|
"entropy": 0.37694089356809857,
|
|
"epoch": 0.07337246531483457,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.07228426933288574,
|
|
"num_tokens": 2605458.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.22220884561538695,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.22220885157585143,
|
|
"step": 550,
|
|
"step_time": 18.586442222911863,
|
|
"zero_adv_skip_rate": 0.4091
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 491.3,
|
|
"completions/max_terminated_length": 339.8,
|
|
"completions/mean_length": 260.625,
|
|
"completions/mean_terminated_length": 195.50928802490233,
|
|
"completions/min_length": 82.0,
|
|
"completions/min_terminated_length": 82.0,
|
|
"entropy": 0.47913156480838853,
|
|
"epoch": 0.07470651013874066,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.8515625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.07808811068534852,
|
|
"num_tokens": 2653796.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.2722742766141891,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.27227428257465364,
|
|
"step": 560,
|
|
"step_time": 11.881868555862457,
|
|
"zero_adv_skip_rate": 0.4089
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 631.5,
|
|
"completions/max_terminated_length": 469.8,
|
|
"completions/mean_length": 227.025,
|
|
"completions/mean_terminated_length": 204.95178833007813,
|
|
"completions/min_length": 81.5,
|
|
"completions/min_terminated_length": 81.5,
|
|
"entropy": 0.43084720000624654,
|
|
"epoch": 0.07604055496264675,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04773980379104614,
|
|
"num_tokens": 2704374.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.22220884561538695,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.22220885157585143,
|
|
"step": 570,
|
|
"step_time": 13.943837157357484,
|
|
"zero_adv_skip_rate": 0.4105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 512.7,
|
|
"completions/max_terminated_length": 502.0,
|
|
"completions/mean_length": 188.75,
|
|
"completions/mean_terminated_length": 186.49464416503906,
|
|
"completions/min_length": 64.7,
|
|
"completions/min_terminated_length": 64.7,
|
|
"entropy": 0.4771064097682635,
|
|
"epoch": 0.07737459978655283,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 1.1015625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.11938934326171875,
|
|
"num_tokens": 2747402.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.14433692693710326,
|
|
"rewards/correctness_reward/mean": 0.2375,
|
|
"rewards/correctness_reward/std": 0.14433693289756774,
|
|
"step": 580,
|
|
"step_time": 11.367576733697206,
|
|
"zero_adv_skip_rate": 0.4155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 507.2,
|
|
"completions/max_terminated_length": 406.8,
|
|
"completions/mean_length": 235.0375,
|
|
"completions/mean_terminated_length": 196.58988418579102,
|
|
"completions/min_length": 79.7,
|
|
"completions/min_terminated_length": 79.7,
|
|
"entropy": 0.4626467996276915,
|
|
"epoch": 0.07870864461045891,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.5078125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.07204353213310241,
|
|
"num_tokens": 2798669.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.4263207823038101,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.42632079422473906,
|
|
"step": 590,
|
|
"step_time": 12.335791666153819,
|
|
"zero_adv_skip_rate": 0.4085
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 599.1,
|
|
"completions/max_terminated_length": 235.7,
|
|
"completions/mean_length": 160.475,
|
|
"completions/mean_terminated_length": 112.26785812377929,
|
|
"completions/min_length": 60.1,
|
|
"completions/min_terminated_length": 60.1,
|
|
"entropy": 0.414915953646414,
|
|
"epoch": 0.08004268943436499,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 1.4765625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1422435760498047,
|
|
"num_tokens": 2844211.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.35561010539531707,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.35561011731624603,
|
|
"step": 600,
|
|
"step_time": 13.839384209644049,
|
|
"zero_adv_skip_rate": 0.405
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 1016.5,
|
|
"completions/max_terminated_length": 689.4,
|
|
"completions/mean_length": 368.2125,
|
|
"completions/mean_terminated_length": 287.7125015258789,
|
|
"completions/min_length": 73.0,
|
|
"completions/min_terminated_length": 73.0,
|
|
"entropy": 0.32835574727505445,
|
|
"epoch": 0.08137673425827108,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.6015625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.007043772935867309,
|
|
"num_tokens": 2904404.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.16875659823417663,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.1687566041946411,
|
|
"step": 610,
|
|
"step_time": 21.759847625903785,
|
|
"zero_adv_skip_rate": 0.4082
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 631.0,
|
|
"completions/max_terminated_length": 629.7,
|
|
"completions/mean_length": 223.525,
|
|
"completions/mean_terminated_length": 222.27857208251953,
|
|
"completions/min_length": 72.9,
|
|
"completions/min_terminated_length": 72.9,
|
|
"entropy": 0.375552400680525,
|
|
"epoch": 0.08271077908217717,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0330017238855362,
|
|
"num_tokens": 2950046.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.2966861009597778,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.29668611884117124,
|
|
"step": 620,
|
|
"step_time": 14.413901050295681,
|
|
"zero_adv_skip_rate": 0.4065
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 502.3,
|
|
"completions/max_terminated_length": 369.3,
|
|
"completions/mean_length": 201.9375,
|
|
"completions/mean_terminated_length": 178.67321853637696,
|
|
"completions/min_length": 85.7,
|
|
"completions/min_terminated_length": 85.7,
|
|
"entropy": 0.35632335785776376,
|
|
"epoch": 0.08404482390608324,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.055638080835342406,
|
|
"num_tokens": 2994393.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.23861626982688905,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.23861626982688905,
|
|
"step": 630,
|
|
"step_time": 11.508118128962815,
|
|
"zero_adv_skip_rate": 0.4079
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 571.6,
|
|
"completions/max_terminated_length": 434.2,
|
|
"completions/mean_length": 229.5125,
|
|
"completions/mean_terminated_length": 206.27500114440917,
|
|
"completions/min_length": 76.1,
|
|
"completions/min_terminated_length": 76.1,
|
|
"entropy": 0.4295649383217096,
|
|
"epoch": 0.08537886872998933,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 1.78125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0009959619492292404,
|
|
"num_tokens": 3042106.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.23691893815994264,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.2369189441204071,
|
|
"step": 640,
|
|
"step_time": 13.06655829893425,
|
|
"zero_adv_skip_rate": 0.4094
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 267.8,
|
|
"completions/max_terminated_length": 235.6,
|
|
"completions/mean_length": 139.125,
|
|
"completions/mean_terminated_length": 133.2250011444092,
|
|
"completions/min_length": 67.4,
|
|
"completions/min_terminated_length": 67.4,
|
|
"entropy": 0.43029848982890445,
|
|
"epoch": 0.0867129135538954,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 1.40625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.12643572092056274,
|
|
"num_tokens": 3080748.0,
|
|
"reward": 0.5375,
|
|
"reward_std": 0.42377244532108305,
|
|
"rewards/correctness_reward/mean": 0.5375,
|
|
"rewards/correctness_reward/std": 0.423772457242012,
|
|
"step": 650,
|
|
"step_time": 7.317079755757004,
|
|
"zero_adv_skip_rate": 0.4046
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 655.8,
|
|
"completions/max_terminated_length": 510.6,
|
|
"completions/mean_length": 349.8875,
|
|
"completions/mean_terminated_length": 273.5400032043457,
|
|
"completions/min_length": 103.2,
|
|
"completions/min_terminated_length": 103.2,
|
|
"entropy": 0.4326310381293297,
|
|
"epoch": 0.08804695837780149,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.78515625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.030124431848526,
|
|
"num_tokens": 3136355.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.1125,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 660,
|
|
"step_time": 14.648396454472095,
|
|
"zero_adv_skip_rate": 0.4061
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 925.8,
|
|
"completions/max_terminated_length": 603.5,
|
|
"completions/mean_length": 393.7,
|
|
"completions/mean_terminated_length": 287.50286254882815,
|
|
"completions/min_length": 114.8,
|
|
"completions/min_terminated_length": 114.8,
|
|
"entropy": 0.3016854617744684,
|
|
"epoch": 0.08938100320170758,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5703125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.01637713760137558,
|
|
"num_tokens": 3196139.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.23860842287540435,
|
|
"rewards/correctness_reward/mean": 0.2125,
|
|
"rewards/correctness_reward/std": 0.2386084347963333,
|
|
"step": 670,
|
|
"step_time": 20.235331940371545,
|
|
"zero_adv_skip_rate": 0.4075
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 500.3,
|
|
"completions/max_terminated_length": 500.3,
|
|
"completions/mean_length": 178.6375,
|
|
"completions/mean_terminated_length": 178.6375,
|
|
"completions/min_length": 58.1,
|
|
"completions/min_terminated_length": 58.1,
|
|
"entropy": 0.3047263156622648,
|
|
"epoch": 0.09071504802561366,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0035021837800741197,
|
|
"num_tokens": 3237518.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.26849985122680664,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.2684998571872711,
|
|
"step": 680,
|
|
"step_time": 11.567399220261724,
|
|
"zero_adv_skip_rate": 0.4074
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 611.2,
|
|
"completions/max_terminated_length": 578.4,
|
|
"completions/mean_length": 248.1375,
|
|
"completions/mean_terminated_length": 228.84642944335937,
|
|
"completions/min_length": 80.7,
|
|
"completions/min_terminated_length": 80.7,
|
|
"entropy": 0.32981937533865374,
|
|
"epoch": 0.09204909284951975,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.28515625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.026255524158477782,
|
|
"num_tokens": 3284841.0,
|
|
"reward": 0.475,
|
|
"reward_std": 0.2575641840696335,
|
|
"rewards/correctness_reward/mean": 0.475,
|
|
"rewards/correctness_reward/std": 0.25756419003009795,
|
|
"step": 690,
|
|
"step_time": 13.880137453507633,
|
|
"zero_adv_skip_rate": 0.4072
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 435.8,
|
|
"completions/max_terminated_length": 430.0,
|
|
"completions/mean_length": 244.3875,
|
|
"completions/mean_terminated_length": 243.01250076293945,
|
|
"completions/min_length": 118.9,
|
|
"completions/min_terminated_length": 118.9,
|
|
"entropy": 0.461271519107478,
|
|
"epoch": 0.09338313767342583,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.046641862392425536,
|
|
"num_tokens": 3335432.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.3021657049655914,
|
|
"rewards/correctness_reward/mean": 0.2,
|
|
"rewards/correctness_reward/std": 0.3021657049655914,
|
|
"step": 700,
|
|
"step_time": 10.466435283329337,
|
|
"zero_adv_skip_rate": 0.4057
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 859.7,
|
|
"completions/max_terminated_length": 279.8,
|
|
"completions/mean_length": 236.9625,
|
|
"completions/mean_terminated_length": 155.5660743713379,
|
|
"completions/min_length": 84.9,
|
|
"completions/min_terminated_length": 84.9,
|
|
"entropy": 0.3359798549984892,
|
|
"epoch": 0.09471718249733191,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.16141281127929688,
|
|
"num_tokens": 3392253.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.2794355183839798,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.2794355243444443,
|
|
"step": 710,
|
|
"step_time": 18.838716050144285,
|
|
"zero_adv_skip_rate": 0.4056
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 615.2,
|
|
"completions/max_terminated_length": 420.8,
|
|
"completions/mean_length": 220.275,
|
|
"completions/mean_terminated_length": 195.33035736083986,
|
|
"completions/min_length": 92.9,
|
|
"completions/min_terminated_length": 92.9,
|
|
"entropy": 0.5872142283866802,
|
|
"epoch": 0.096051227321238,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.20842809677124025,
|
|
"num_tokens": 3440091.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.11700168251991272,
|
|
"rewards/correctness_reward/mean": 0.225,
|
|
"rewards/correctness_reward/std": 0.11700168251991272,
|
|
"step": 720,
|
|
"step_time": 13.514335963316261,
|
|
"zero_adv_skip_rate": 0.4097
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 489.8,
|
|
"completions/max_terminated_length": 317.5,
|
|
"completions/mean_length": 195.7625,
|
|
"completions/mean_terminated_length": 171.16964416503907,
|
|
"completions/min_length": 76.9,
|
|
"completions/min_terminated_length": 76.9,
|
|
"entropy": 0.3741204805805215,
|
|
"epoch": 0.09738527214514407,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.010610587894916534,
|
|
"num_tokens": 3491480.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.336654344201088,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.3366543620824814,
|
|
"step": 730,
|
|
"step_time": 11.588704235479236,
|
|
"zero_adv_skip_rate": 0.4082
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 421.8,
|
|
"completions/max_terminated_length": 387.6,
|
|
"completions/mean_length": 189.1625,
|
|
"completions/mean_terminated_length": 183.45000152587892,
|
|
"completions/min_length": 74.8,
|
|
"completions/min_terminated_length": 74.8,
|
|
"entropy": 0.46430945050503525,
|
|
"epoch": 0.09871931696905016,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.56640625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.08759346008300781,
|
|
"num_tokens": 3538133.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.28029437065124513,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.28029437065124513,
|
|
"step": 740,
|
|
"step_time": 10.21667982749641,
|
|
"zero_adv_skip_rate": 0.4068
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 459.6,
|
|
"completions/max_terminated_length": 287.9,
|
|
"completions/mean_length": 187.9125,
|
|
"completions/mean_terminated_length": 164.9125015258789,
|
|
"completions/min_length": 87.2,
|
|
"completions/min_terminated_length": 87.2,
|
|
"entropy": 0.3842765107750893,
|
|
"epoch": 0.10005336179295625,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.78515625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.053933990001678464,
|
|
"num_tokens": 3582030.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.22051936089992524,
|
|
"rewards/correctness_reward/mean": 0.2125,
|
|
"rewards/correctness_reward/std": 0.22051936089992524,
|
|
"step": 750,
|
|
"step_time": 10.78542556334287,
|
|
"zero_adv_skip_rate": 0.408
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 565.4,
|
|
"completions/max_terminated_length": 529.2,
|
|
"completions/mean_length": 214.775,
|
|
"completions/mean_terminated_length": 191.91428833007814,
|
|
"completions/min_length": 92.7,
|
|
"completions/min_terminated_length": 92.7,
|
|
"entropy": 0.411581905026521,
|
|
"epoch": 0.10138740661686232,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.013198897242546082,
|
|
"num_tokens": 3629516.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.3093190997838974,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.30931911170482634,
|
|
"step": 760,
|
|
"step_time": 13.150145673658699,
|
|
"zero_adv_skip_rate": 0.4066
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 394.9,
|
|
"completions/max_terminated_length": 189.4,
|
|
"completions/mean_length": 131.95,
|
|
"completions/mean_terminated_length": 103.24464492797851,
|
|
"completions/min_length": 57.0,
|
|
"completions/min_terminated_length": 57.0,
|
|
"entropy": 0.3981681330395596,
|
|
"epoch": 0.10272145144076841,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.66796875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.00679100751876831,
|
|
"num_tokens": 3666464.0,
|
|
"reward": 0.5625,
|
|
"reward_std": 0.33495701253414156,
|
|
"rewards/correctness_reward/mean": 0.5625,
|
|
"rewards/correctness_reward/std": 0.33495703637599944,
|
|
"step": 770,
|
|
"step_time": 9.631225976906716,
|
|
"zero_adv_skip_rate": 0.4052
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 710.6,
|
|
"completions/max_terminated_length": 452.5,
|
|
"completions/mean_length": 261.975,
|
|
"completions/mean_terminated_length": 218.56964721679688,
|
|
"completions/min_length": 80.9,
|
|
"completions/min_terminated_length": 80.9,
|
|
"entropy": 0.38883779053058887,
|
|
"epoch": 0.10405549626467449,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.482421875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.05006141662597656,
|
|
"num_tokens": 3714046.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.40020377933979034,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.40020379722118377,
|
|
"step": 780,
|
|
"step_time": 16.293143813032657,
|
|
"zero_adv_skip_rate": 0.4013
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 498.7,
|
|
"completions/max_terminated_length": 300.2,
|
|
"completions/mean_length": 160.9125,
|
|
"completions/mean_terminated_length": 133.80714492797853,
|
|
"completions/min_length": 64.9,
|
|
"completions/min_terminated_length": 64.9,
|
|
"entropy": 0.4593452331610024,
|
|
"epoch": 0.10538954108858058,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.047008955478668214,
|
|
"num_tokens": 3754719.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.3518356800079346,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.35183569192886355,
|
|
"step": 790,
|
|
"step_time": 11.847902400698512,
|
|
"zero_adv_skip_rate": 0.3987
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 541.7,
|
|
"completions/max_terminated_length": 541.7,
|
|
"completions/mean_length": 208.3125,
|
|
"completions/mean_terminated_length": 208.3125,
|
|
"completions/min_length": 99.9,
|
|
"completions/min_terminated_length": 99.9,
|
|
"entropy": 0.415580485171328,
|
|
"epoch": 0.10672358591248667,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.78515625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.012733949720859528,
|
|
"num_tokens": 3801528.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.2504029482603073,
|
|
"step": 800,
|
|
"step_time": 12.435840690601617,
|
|
"zero_adv_skip_rate": 0.3987
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 939.7,
|
|
"completions/max_terminated_length": 402.8,
|
|
"completions/mean_length": 233.925,
|
|
"completions/mean_terminated_length": 160.97321701049805,
|
|
"completions/min_length": 72.3,
|
|
"completions/min_terminated_length": 72.3,
|
|
"entropy": 0.41899712414791185,
|
|
"epoch": 0.10805763073639274,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.060791182518005374,
|
|
"num_tokens": 3847506.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.2886738538742065,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.2886738657951355,
|
|
"step": 810,
|
|
"step_time": 20.489199703279883,
|
|
"zero_adv_skip_rate": 0.3988
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 436.7,
|
|
"completions/max_terminated_length": 245.9,
|
|
"completions/mean_length": 144.5875,
|
|
"completions/mean_terminated_length": 119.34107284545898,
|
|
"completions/min_length": 55.1,
|
|
"completions/min_terminated_length": 55.1,
|
|
"entropy": 0.41902368143200874,
|
|
"epoch": 0.10939167556029883,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.023132836818695067,
|
|
"num_tokens": 3886065.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.33751319646835326,
|
|
"rewards/correctness_reward/mean": 0.45,
|
|
"rewards/correctness_reward/std": 0.3375132083892822,
|
|
"step": 820,
|
|
"step_time": 10.709570042975248,
|
|
"zero_adv_skip_rate": 0.3963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 803.5,
|
|
"completions/max_terminated_length": 477.4,
|
|
"completions/mean_length": 252.0125,
|
|
"completions/mean_terminated_length": 183.9607192993164,
|
|
"completions/min_length": 65.9,
|
|
"completions/min_terminated_length": 65.9,
|
|
"entropy": 0.4549306676323925,
|
|
"epoch": 0.1107257203842049,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0982884407043457,
|
|
"num_tokens": 3935114.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.3021578580141068,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.3021578699350357,
|
|
"step": 830,
|
|
"step_time": 18.14434881694615,
|
|
"zero_adv_skip_rate": 0.3952
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 629.9,
|
|
"completions/max_terminated_length": 519.7,
|
|
"completions/mean_length": 236.9625,
|
|
"completions/mean_terminated_length": 219.05178680419922,
|
|
"completions/min_length": 80.8,
|
|
"completions/min_terminated_length": 80.8,
|
|
"entropy": 0.346521221101284,
|
|
"epoch": 0.11205976520811099,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.671875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.00805482268333435,
|
|
"num_tokens": 3988583.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.16875659823417663,
|
|
"rewards/correctness_reward/mean": 0.1125,
|
|
"rewards/correctness_reward/std": 0.1687566041946411,
|
|
"step": 840,
|
|
"step_time": 14.141272801812738,
|
|
"zero_adv_skip_rate": 0.3976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 786.1,
|
|
"completions/max_terminated_length": 422.6,
|
|
"completions/mean_length": 264.225,
|
|
"completions/mean_terminated_length": 178.57143020629883,
|
|
"completions/min_length": 78.1,
|
|
"completions/min_terminated_length": 78.1,
|
|
"entropy": 0.42856587030525717,
|
|
"epoch": 0.11339381003201708,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.11787569522857666,
|
|
"num_tokens": 4037281.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.29838343262672423,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.2983834445476532,
|
|
"step": 850,
|
|
"step_time": 17.80626538041979,
|
|
"zero_adv_skip_rate": 0.3965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 503.2,
|
|
"completions/max_terminated_length": 350.4,
|
|
"completions/mean_length": 215.0375,
|
|
"completions/mean_terminated_length": 192.86607360839844,
|
|
"completions/min_length": 82.6,
|
|
"completions/min_terminated_length": 82.6,
|
|
"entropy": 0.37352889163109165,
|
|
"epoch": 0.11472785485592316,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.376953125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.017108671367168427,
|
|
"num_tokens": 4082780.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.26133076250553133,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.26133078038692475,
|
|
"step": 860,
|
|
"step_time": 11.869846995454282,
|
|
"zero_adv_skip_rate": 0.3965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 625.4,
|
|
"completions/max_terminated_length": 265.4,
|
|
"completions/mean_length": 186.4125,
|
|
"completions/mean_terminated_length": 138.74464492797853,
|
|
"completions/min_length": 67.9,
|
|
"completions/min_terminated_length": 67.9,
|
|
"entropy": 0.44138020230457187,
|
|
"epoch": 0.11606189967982924,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0048992447555065155,
|
|
"num_tokens": 4125685.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.3593845307826996,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.3593845427036285,
|
|
"step": 870,
|
|
"step_time": 14.750373977702111,
|
|
"zero_adv_skip_rate": 0.3943
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 670.0,
|
|
"completions/max_terminated_length": 498.9,
|
|
"completions/mean_length": 253.0875,
|
|
"completions/mean_terminated_length": 227.003572845459,
|
|
"completions/min_length": 92.5,
|
|
"completions/min_terminated_length": 92.5,
|
|
"entropy": 0.4585113919207028,
|
|
"epoch": 0.11739594450373532,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.12974621057510377,
|
|
"num_tokens": 4175572.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.29838343262672423,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.2983834445476532,
|
|
"step": 880,
|
|
"step_time": 15.5907517448999,
|
|
"zero_adv_skip_rate": 0.3932
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 465.8,
|
|
"completions/max_terminated_length": 453.6,
|
|
"completions/mean_length": 218.675,
|
|
"completions/mean_terminated_length": 214.5285743713379,
|
|
"completions/min_length": 109.2,
|
|
"completions/min_terminated_length": 109.2,
|
|
"entropy": 0.4569976143538952,
|
|
"epoch": 0.11872998932764141,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.011200755834579468,
|
|
"num_tokens": 4227746.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.1960918426513672,
|
|
"rewards/correctness_reward/mean": 0.225,
|
|
"rewards/correctness_reward/std": 0.19609185457229614,
|
|
"step": 890,
|
|
"step_time": 10.846475318353622,
|
|
"zero_adv_skip_rate": 0.3955
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 814.7,
|
|
"completions/max_terminated_length": 442.3,
|
|
"completions/mean_length": 255.7,
|
|
"completions/mean_terminated_length": 204.55893096923828,
|
|
"completions/min_length": 64.4,
|
|
"completions/min_terminated_length": 64.4,
|
|
"entropy": 0.3978612648323178,
|
|
"epoch": 0.1200640341515475,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0005871027708053589,
|
|
"num_tokens": 4277026.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.23861626982688905,
|
|
"rewards/correctness_reward/mean": 0.15,
|
|
"rewards/correctness_reward/std": 0.23861626982688905,
|
|
"step": 900,
|
|
"step_time": 18.121651736367493,
|
|
"zero_adv_skip_rate": 0.3967
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 246.2,
|
|
"completions/max_terminated_length": 243.6,
|
|
"completions/mean_length": 129.0875,
|
|
"completions/mean_terminated_length": 126.6732162475586,
|
|
"completions/min_length": 64.0,
|
|
"completions/min_terminated_length": 64.0,
|
|
"entropy": 0.464518169251581,
|
|
"epoch": 0.12139807897545357,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.7421875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.053516602516174315,
|
|
"num_tokens": 4313377.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.28659676015377045,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.2865967661142349,
|
|
"step": 910,
|
|
"step_time": 6.534653309825808,
|
|
"zero_adv_skip_rate": 0.3967
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 452.9,
|
|
"completions/max_terminated_length": 441.2,
|
|
"completions/mean_length": 184.6875,
|
|
"completions/mean_terminated_length": 180.34285888671874,
|
|
"completions/min_length": 66.8,
|
|
"completions/min_terminated_length": 66.8,
|
|
"entropy": 0.4865027226507664,
|
|
"epoch": 0.12273212379935966,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.004426823556423187,
|
|
"num_tokens": 4361368.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.2125,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 920,
|
|
"step_time": 10.585470244847238,
|
|
"zero_adv_skip_rate": 0.3978
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 858.0,
|
|
"completions/max_terminated_length": 373.1,
|
|
"completions/mean_length": 260.4125,
|
|
"completions/mean_terminated_length": 194.31250686645507,
|
|
"completions/min_length": 92.8,
|
|
"completions/min_terminated_length": 92.8,
|
|
"entropy": 0.5379007135052234,
|
|
"epoch": 0.12406616862326574,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.08550215363502503,
|
|
"num_tokens": 4411233.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.3610740154981613,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.36107403337955474,
|
|
"step": 930,
|
|
"step_time": 19.447785599250345,
|
|
"zero_adv_skip_rate": 0.3957
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 736.4,
|
|
"completions/max_terminated_length": 352.8,
|
|
"completions/mean_length": 186.5,
|
|
"completions/mean_terminated_length": 137.22143020629883,
|
|
"completions/min_length": 63.2,
|
|
"completions/min_terminated_length": 63.2,
|
|
"entropy": 0.35257831853814425,
|
|
"epoch": 0.12540021344717184,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.490234375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.13315120935440064,
|
|
"num_tokens": 4452713.0,
|
|
"reward": 0.5375,
|
|
"reward_std": 0.5096362382173538,
|
|
"rewards/correctness_reward/mean": 0.5375,
|
|
"rewards/correctness_reward/std": 0.5096362620592118,
|
|
"step": 940,
|
|
"step_time": 17.173778134211897,
|
|
"zero_adv_skip_rate": 0.3915
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 292.4,
|
|
"completions/max_terminated_length": 291.5,
|
|
"completions/mean_length": 171.4125,
|
|
"completions/mean_terminated_length": 171.00535736083984,
|
|
"completions/min_length": 102.2,
|
|
"completions/min_terminated_length": 102.2,
|
|
"entropy": 0.34987218988438445,
|
|
"epoch": 0.1267342582710779,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.62109375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.028108441829681398,
|
|
"num_tokens": 4493866.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.36363019943237307,
|
|
"rewards/correctness_reward/mean": 0.4375,
|
|
"rewards/correctness_reward/std": 0.3636302053928375,
|
|
"step": 950,
|
|
"step_time": 7.8882073177956045,
|
|
"zero_adv_skip_rate": 0.3884
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 655.1,
|
|
"completions/max_terminated_length": 477.3,
|
|
"completions/mean_length": 216.55,
|
|
"completions/mean_terminated_length": 192.6,
|
|
"completions/min_length": 83.0,
|
|
"completions/min_terminated_length": 83.0,
|
|
"entropy": 0.37622416571580936,
|
|
"epoch": 0.128068303094984,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.32421875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.006172598898410797,
|
|
"num_tokens": 4540318.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.33288776576519014,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.3328877717256546,
|
|
"step": 960,
|
|
"step_time": 15.058704853616655,
|
|
"zero_adv_skip_rate": 0.3875
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 450.5,
|
|
"completions/max_terminated_length": 449.5,
|
|
"completions/mean_length": 177.4875,
|
|
"completions/mean_terminated_length": 177.1196434020996,
|
|
"completions/min_length": 76.3,
|
|
"completions/min_terminated_length": 76.3,
|
|
"entropy": 0.3228327240794897,
|
|
"epoch": 0.12940234791889008,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.029935294389724733,
|
|
"num_tokens": 4583253.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.2739637613296509,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.27396377325057986,
|
|
"step": 970,
|
|
"step_time": 10.775660025049001,
|
|
"zero_adv_skip_rate": 0.3876
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 726.8,
|
|
"completions/max_terminated_length": 649.1,
|
|
"completions/mean_length": 284.3,
|
|
"completions/mean_terminated_length": 254.46250762939454,
|
|
"completions/min_length": 85.2,
|
|
"completions/min_terminated_length": 85.2,
|
|
"entropy": 0.45240020295605066,
|
|
"epoch": 0.13073639274279616,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.023830027878284456,
|
|
"num_tokens": 4634341.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.23144718110561371,
|
|
"rewards/correctness_reward/mean": 0.4375,
|
|
"rewards/correctness_reward/std": 0.23144719302654265,
|
|
"step": 980,
|
|
"step_time": 16.26325493603945,
|
|
"zero_adv_skip_rate": 0.3888
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 550.0,
|
|
"completions/max_terminated_length": 521.0,
|
|
"completions/mean_length": 191.7,
|
|
"completions/mean_terminated_length": 171.7107162475586,
|
|
"completions/min_length": 56.8,
|
|
"completions/min_terminated_length": 56.8,
|
|
"entropy": 0.38459187373518944,
|
|
"epoch": 0.13207043756670225,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.046795278787612915,
|
|
"num_tokens": 4676749.0,
|
|
"reward": 0.525,
|
|
"reward_std": 0.13340125977993011,
|
|
"rewards/correctness_reward/mean": 0.525,
|
|
"rewards/correctness_reward/std": 0.1334012657403946,
|
|
"step": 990,
|
|
"step_time": 12.439414240140469,
|
|
"zero_adv_skip_rate": 0.3919
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 903.9,
|
|
"completions/max_terminated_length": 459.7,
|
|
"completions/mean_length": 320.825,
|
|
"completions/mean_terminated_length": 254.38750343322755,
|
|
"completions/min_length": 124.1,
|
|
"completions/min_terminated_length": 124.1,
|
|
"entropy": 0.23718904874597987,
|
|
"epoch": 0.13340448239060831,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.043797779083251956,
|
|
"num_tokens": 4732983.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.12793734967708587,
|
|
"rewards/correctness_reward/mean": 0.2375,
|
|
"rewards/correctness_reward/std": 0.12793734967708587,
|
|
"step": 1000,
|
|
"step_time": 19.535986811853945,
|
|
"zero_adv_skip_rate": 0.395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 687.8,
|
|
"completions/max_terminated_length": 544.1,
|
|
"completions/mean_length": 210.2,
|
|
"completions/mean_terminated_length": 188.6678596496582,
|
|
"completions/min_length": 84.7,
|
|
"completions/min_terminated_length": 84.7,
|
|
"entropy": 0.4302984630422933,
|
|
"epoch": 0.1347385272145144,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.32521615028381345,
|
|
"num_tokens": 4777151.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.2966939479112625,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.296693953871727,
|
|
"step": 1010,
|
|
"step_time": 16.82924981517717,
|
|
"zero_adv_skip_rate": 0.3941
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 705.9,
|
|
"completions/max_terminated_length": 529.6,
|
|
"completions/mean_length": 294.6875,
|
|
"completions/mean_terminated_length": 265.9839324951172,
|
|
"completions/min_length": 102.6,
|
|
"completions/min_terminated_length": 102.6,
|
|
"entropy": 0.4209234733134508,
|
|
"epoch": 0.1360725720384205,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.05003441572189331,
|
|
"num_tokens": 4839734.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.1498008370399475,
|
|
"rewards/correctness_reward/mean": 0.1,
|
|
"rewards/correctness_reward/std": 0.14980084896087648,
|
|
"step": 1020,
|
|
"step_time": 15.679908427130432,
|
|
"zero_adv_skip_rate": 0.3971
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 898.0,
|
|
"completions/max_terminated_length": 492.8,
|
|
"completions/mean_length": 281.65,
|
|
"completions/mean_terminated_length": 214.3750030517578,
|
|
"completions/min_length": 70.6,
|
|
"completions/min_terminated_length": 70.6,
|
|
"entropy": 0.45002511143684387,
|
|
"epoch": 0.13740661686232658,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.88671875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.05603691339492798,
|
|
"num_tokens": 4891674.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.2722664296627045,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.2722664475440979,
|
|
"step": 1030,
|
|
"step_time": 20.711928214412183,
|
|
"zero_adv_skip_rate": 0.3971
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 283.0,
|
|
"completions/max_terminated_length": 259.5,
|
|
"completions/mean_length": 153.75,
|
|
"completions/mean_terminated_length": 148.95714416503907,
|
|
"completions/min_length": 68.4,
|
|
"completions/min_terminated_length": 68.4,
|
|
"entropy": 0.4069023351185024,
|
|
"epoch": 0.13874066168623267,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.017635151743888855,
|
|
"num_tokens": 4931590.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.1960918426513672,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.19609185457229614,
|
|
"step": 1040,
|
|
"step_time": 7.345876767672598,
|
|
"zero_adv_skip_rate": 0.399
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 681.4,
|
|
"completions/max_terminated_length": 383.8,
|
|
"completions/mean_length": 255.9375,
|
|
"completions/mean_terminated_length": 209.91964721679688,
|
|
"completions/min_length": 89.1,
|
|
"completions/min_terminated_length": 89.1,
|
|
"entropy": 0.36115679144859314,
|
|
"epoch": 0.14007470651013873,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.33984375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.14281668663024902,
|
|
"num_tokens": 4980017.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.2941299170255661,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.2941299468278885,
|
|
"step": 1050,
|
|
"step_time": 16.286271713580938,
|
|
"zero_adv_skip_rate": 0.399
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 505.7,
|
|
"completions/max_terminated_length": 325.7,
|
|
"completions/mean_length": 163.5625,
|
|
"completions/mean_terminated_length": 139.86785774230958,
|
|
"completions/min_length": 78.2,
|
|
"completions/min_terminated_length": 78.2,
|
|
"entropy": 0.36780036333948374,
|
|
"epoch": 0.14140875133404482,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.05860264897346497,
|
|
"num_tokens": 5019542.0,
|
|
"reward": 0.5125,
|
|
"reward_std": 0.261338609457016,
|
|
"rewards/correctness_reward/mean": 0.5125,
|
|
"rewards/correctness_reward/std": 0.2613386154174805,
|
|
"step": 1060,
|
|
"step_time": 12.08125509256497,
|
|
"zero_adv_skip_rate": 0.3991
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 251.4,
|
|
"completions/max_terminated_length": 248.0,
|
|
"completions/mean_length": 136.9,
|
|
"completions/mean_terminated_length": 135.8125015258789,
|
|
"completions/min_length": 77.5,
|
|
"completions/min_terminated_length": 77.5,
|
|
"entropy": 0.3318399537820369,
|
|
"epoch": 0.1427427961579509,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.077503502368927,
|
|
"num_tokens": 5061958.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.3518356800079346,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.35183569192886355,
|
|
"step": 1070,
|
|
"step_time": 6.982704845443368,
|
|
"zero_adv_skip_rate": 0.3972
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 515.4,
|
|
"completions/max_terminated_length": 496.3,
|
|
"completions/mean_length": 184.2375,
|
|
"completions/mean_terminated_length": 177.7678596496582,
|
|
"completions/min_length": 61.0,
|
|
"completions/min_terminated_length": 61.0,
|
|
"entropy": 0.4872568041086197,
|
|
"epoch": 0.144076840981857,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.040798038244247437,
|
|
"num_tokens": 5103833.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.22390617728233336,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.22390617728233336,
|
|
"step": 1080,
|
|
"step_time": 12.000737405754625,
|
|
"zero_adv_skip_rate": 0.3981
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 730.0,
|
|
"completions/max_terminated_length": 561.6,
|
|
"completions/mean_length": 229.325,
|
|
"completions/mean_terminated_length": 200.86607208251954,
|
|
"completions/min_length": 70.4,
|
|
"completions/min_terminated_length": 70.4,
|
|
"entropy": 0.5097680293256417,
|
|
"epoch": 0.14541088580576308,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.390625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0453015148639679,
|
|
"num_tokens": 5150011.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.38086825609207153,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.3808682680130005,
|
|
"step": 1090,
|
|
"step_time": 16.871693007834256,
|
|
"zero_adv_skip_rate": 0.3963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 526.8,
|
|
"completions/max_terminated_length": 526.7,
|
|
"completions/mean_length": 219.7375,
|
|
"completions/mean_terminated_length": 216.4982147216797,
|
|
"completions/min_length": 87.4,
|
|
"completions/min_terminated_length": 87.4,
|
|
"entropy": 0.4559941159871717,
|
|
"epoch": 0.14674493062966915,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.025596034526824952,
|
|
"num_tokens": 5200102.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.2937580019235611,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.29375800788402556,
|
|
"step": 1100,
|
|
"step_time": 12.446640572696925,
|
|
"zero_adv_skip_rate": 0.3964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 525.2,
|
|
"completions/max_terminated_length": 365.2,
|
|
"completions/mean_length": 191.5875,
|
|
"completions/mean_terminated_length": 168.91250228881836,
|
|
"completions/min_length": 63.6,
|
|
"completions/min_terminated_length": 63.6,
|
|
"entropy": 0.37868189505700556,
|
|
"epoch": 0.14807897545357523,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.65234375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.12362277507781982,
|
|
"num_tokens": 5243693.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.28029437065124513,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.28029437065124513,
|
|
"step": 1110,
|
|
"step_time": 12.473342481534928,
|
|
"zero_adv_skip_rate": 0.3955
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 375.5,
|
|
"completions/max_terminated_length": 256.2,
|
|
"completions/mean_length": 153.8,
|
|
"completions/mean_terminated_length": 131.8160743713379,
|
|
"completions/min_length": 66.9,
|
|
"completions/min_terminated_length": 66.9,
|
|
"entropy": 0.37764241608480614,
|
|
"epoch": 0.14941302027748132,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.71484375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.01362324059009552,
|
|
"num_tokens": 5282797.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.3653275310993195,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.3653275310993195,
|
|
"step": 1120,
|
|
"step_time": 9.92089770073071,
|
|
"zero_adv_skip_rate": 0.3929
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 419.3,
|
|
"completions/max_terminated_length": 416.7,
|
|
"completions/mean_length": 215.625,
|
|
"completions/mean_terminated_length": 214.36785736083985,
|
|
"completions/min_length": 102.2,
|
|
"completions/min_terminated_length": 102.2,
|
|
"entropy": 0.37504677647458656,
|
|
"epoch": 0.1507470651013874,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.013964718580245972,
|
|
"num_tokens": 5327495.0,
|
|
"reward": 0.5375,
|
|
"reward_std": 0.33287991881370543,
|
|
"rewards/correctness_reward/mean": 0.5375,
|
|
"rewards/correctness_reward/std": 0.33287993669509885,
|
|
"step": 1130,
|
|
"step_time": 10.68902911124751,
|
|
"zero_adv_skip_rate": 0.392
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 319.1,
|
|
"completions/max_terminated_length": 317.2,
|
|
"completions/mean_length": 137.8125,
|
|
"completions/mean_terminated_length": 137.19107208251953,
|
|
"completions/min_length": 58.7,
|
|
"completions/min_terminated_length": 58.7,
|
|
"entropy": 0.37427876361956197,
|
|
"epoch": 0.1520811099252935,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.01694630682468414,
|
|
"num_tokens": 5365040.0,
|
|
"reward": 0.5375,
|
|
"reward_std": 0.23022893965244293,
|
|
"rewards/correctness_reward/mean": 0.5375,
|
|
"rewards/correctness_reward/std": 0.23022893965244293,
|
|
"step": 1140,
|
|
"step_time": 8.534799864981323,
|
|
"zero_adv_skip_rate": 0.3921
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 472.5,
|
|
"completions/max_terminated_length": 438.8,
|
|
"completions/mean_length": 175.4625,
|
|
"completions/mean_terminated_length": 168.01071548461914,
|
|
"completions/min_length": 68.7,
|
|
"completions/min_terminated_length": 68.7,
|
|
"entropy": 0.5109803112489837,
|
|
"epoch": 0.15341515474919956,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.439453125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.11647236347198486,
|
|
"num_tokens": 5408573.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.29838343262672423,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.2983834445476532,
|
|
"step": 1150,
|
|
"step_time": 11.944715827330947,
|
|
"zero_adv_skip_rate": 0.3913
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 345.5,
|
|
"completions/max_terminated_length": 337.9,
|
|
"completions/mean_length": 168.475,
|
|
"completions/mean_terminated_length": 166.8142868041992,
|
|
"completions/min_length": 85.1,
|
|
"completions/min_terminated_length": 85.1,
|
|
"entropy": 0.3815567764852728,
|
|
"epoch": 0.15474919957310565,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.59765625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0018134694546461105,
|
|
"num_tokens": 5449595.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.33287991881370543,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.33287993669509885,
|
|
"step": 1160,
|
|
"step_time": 9.09448106009513,
|
|
"zero_adv_skip_rate": 0.3905
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 285.3,
|
|
"completions/max_terminated_length": 280.2,
|
|
"completions/mean_length": 138.7625,
|
|
"completions/mean_terminated_length": 137.69464378356935,
|
|
"completions/min_length": 68.5,
|
|
"completions/min_terminated_length": 68.5,
|
|
"entropy": 0.3875306240682091,
|
|
"epoch": 0.15608324439701174,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.058624500036239625,
|
|
"num_tokens": 5487664.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.3257186770439148,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.32571869492530825,
|
|
"step": 1170,
|
|
"step_time": 7.536423017457127,
|
|
"zero_adv_skip_rate": 0.3897
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 348.6,
|
|
"completions/max_terminated_length": 341.6,
|
|
"completions/mean_length": 144.85,
|
|
"completions/mean_terminated_length": 142.20357208251954,
|
|
"completions/min_length": 65.3,
|
|
"completions/min_terminated_length": 65.3,
|
|
"entropy": 0.40871678239532877,
|
|
"epoch": 0.15741728922091783,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.8203125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.005516320466995239,
|
|
"num_tokens": 5529300.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.2912300378084183,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.2912300378084183,
|
|
"step": 1180,
|
|
"step_time": 8.903724123444409,
|
|
"zero_adv_skip_rate": 0.389
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 473.0,
|
|
"completions/max_terminated_length": 331.0,
|
|
"completions/mean_length": 190.1375,
|
|
"completions/mean_terminated_length": 169.55000114440918,
|
|
"completions/min_length": 81.4,
|
|
"completions/min_terminated_length": 81.4,
|
|
"entropy": 0.37728561740368605,
|
|
"epoch": 0.15875133404482392,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.75,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0023335158824920655,
|
|
"num_tokens": 5573151.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.3282748609781265,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.328274866938591,
|
|
"step": 1190,
|
|
"step_time": 11.825452901143581,
|
|
"zero_adv_skip_rate": 0.3874
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 669.8,
|
|
"completions/max_terminated_length": 439.7,
|
|
"completions/mean_length": 267.8,
|
|
"completions/mean_terminated_length": 218.46072387695312,
|
|
"completions/min_length": 89.1,
|
|
"completions/min_terminated_length": 89.1,
|
|
"entropy": 0.3565249485628946,
|
|
"epoch": 0.16008537886872998,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.236328125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04380530416965485,
|
|
"num_tokens": 5622495.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.3127137631177902,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.3127137631177902,
|
|
"step": 1200,
|
|
"step_time": 16.532186142820866,
|
|
"zero_adv_skip_rate": 0.3867
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 601.1,
|
|
"completions/max_terminated_length": 416.6,
|
|
"completions/mean_length": 221.4875,
|
|
"completions/mean_terminated_length": 197.67857208251954,
|
|
"completions/min_length": 82.8,
|
|
"completions/min_terminated_length": 82.8,
|
|
"entropy": 0.47183349262923,
|
|
"epoch": 0.16141942369263607,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 1.65625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.09921945929527283,
|
|
"num_tokens": 5672190.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.321952098608017,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.32195210456848145,
|
|
"step": 1210,
|
|
"step_time": 14.092544680461287,
|
|
"zero_adv_skip_rate": 0.386
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 539.3,
|
|
"completions/max_terminated_length": 516.7,
|
|
"completions/mean_length": 226.4,
|
|
"completions/mean_terminated_length": 222.45000038146972,
|
|
"completions/min_length": 86.0,
|
|
"completions/min_terminated_length": 86.0,
|
|
"entropy": 0.4872974008321762,
|
|
"epoch": 0.16275346851654215,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.010899662971496582,
|
|
"num_tokens": 5718974.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.0980459213256836,
|
|
"rewards/correctness_reward/mean": 0.3375,
|
|
"rewards/correctness_reward/std": 0.09804592728614807,
|
|
"step": 1220,
|
|
"step_time": 12.091492725722492,
|
|
"zero_adv_skip_rate": 0.3893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 598.7,
|
|
"completions/max_terminated_length": 416.2,
|
|
"completions/mean_length": 251.2,
|
|
"completions/mean_terminated_length": 227.1696434020996,
|
|
"completions/min_length": 103.2,
|
|
"completions/min_terminated_length": 103.2,
|
|
"entropy": 0.4420344614190981,
|
|
"epoch": 0.16408751334044824,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.3671875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.00820053592324257,
|
|
"num_tokens": 5767934.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.37031235098838805,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.370312374830246,
|
|
"step": 1230,
|
|
"step_time": 14.17843578113243,
|
|
"zero_adv_skip_rate": 0.3878
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 302.9,
|
|
"completions/max_terminated_length": 295.6,
|
|
"completions/mean_length": 157.2,
|
|
"completions/mean_terminated_length": 155.78750076293946,
|
|
"completions/min_length": 70.5,
|
|
"completions/min_terminated_length": 70.5,
|
|
"entropy": 0.3631798837866102,
|
|
"epoch": 0.16542155816435433,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.87109375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.008543021976947784,
|
|
"num_tokens": 5807830.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.30762176811695097,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.30762178599834444,
|
|
"step": 1240,
|
|
"step_time": 7.860969302430749,
|
|
"zero_adv_skip_rate": 0.3871
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 375.0,
|
|
"completions/max_terminated_length": 374.2,
|
|
"completions/mean_length": 172.3,
|
|
"completions/mean_terminated_length": 172.0375,
|
|
"completions/min_length": 63.3,
|
|
"completions/min_terminated_length": 63.3,
|
|
"entropy": 0.40907218877691776,
|
|
"epoch": 0.1667556029882604,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.035176506638526915,
|
|
"num_tokens": 5851094.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.3593845307826996,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.3593845427036285,
|
|
"step": 1250,
|
|
"step_time": 9.507945601642131,
|
|
"zero_adv_skip_rate": 0.3856
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 594.9,
|
|
"completions/max_terminated_length": 436.9,
|
|
"completions/mean_length": 236.475,
|
|
"completions/mean_terminated_length": 213.54464416503907,
|
|
"completions/min_length": 105.6,
|
|
"completions/min_terminated_length": 105.6,
|
|
"entropy": 0.39357615182442324,
|
|
"epoch": 0.16808964781216648,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1395635724067688,
|
|
"num_tokens": 5897868.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.269358703494072,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.269358703494072,
|
|
"step": 1260,
|
|
"step_time": 13.920270685851573,
|
|
"zero_adv_skip_rate": 0.3849
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 506.0,
|
|
"completions/max_terminated_length": 427.6,
|
|
"completions/mean_length": 261.0375,
|
|
"completions/mean_terminated_length": 225.08095512390136,
|
|
"completions/min_length": 109.3,
|
|
"completions/min_terminated_length": 109.3,
|
|
"entropy": 0.45746402628719807,
|
|
"epoch": 0.16942369263607257,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.07401443719863891,
|
|
"num_tokens": 5946583.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.24954409003257752,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.24954410195350646,
|
|
"step": 1270,
|
|
"step_time": 11.754356036521495,
|
|
"zero_adv_skip_rate": 0.3858
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 721.4,
|
|
"completions/max_terminated_length": 403.9,
|
|
"completions/mean_length": 258.0875,
|
|
"completions/mean_terminated_length": 211.42678833007812,
|
|
"completions/min_length": 93.4,
|
|
"completions/min_terminated_length": 93.4,
|
|
"entropy": 0.3074905403579275,
|
|
"epoch": 0.17075773745997866,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0093656986951828,
|
|
"num_tokens": 5998174.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.12793734967708587,
|
|
"rewards/correctness_reward/mean": 0.2375,
|
|
"rewards/correctness_reward/std": 0.12793734967708587,
|
|
"step": 1280,
|
|
"step_time": 15.95580695308745,
|
|
"zero_adv_skip_rate": 0.3883
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 638.1,
|
|
"completions/max_terminated_length": 291.3,
|
|
"completions/mean_length": 214.975,
|
|
"completions/mean_terminated_length": 162.44286041259767,
|
|
"completions/min_length": 75.1,
|
|
"completions/min_terminated_length": 75.1,
|
|
"entropy": 0.523714835755527,
|
|
"epoch": 0.17209178228388475,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.5859375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0565889835357666,
|
|
"num_tokens": 6050900.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.2794355183839798,
|
|
"rewards/correctness_reward/mean": 0.2,
|
|
"rewards/correctness_reward/std": 0.2794355243444443,
|
|
"step": 1290,
|
|
"step_time": 14.81226703999564,
|
|
"zero_adv_skip_rate": 0.3884
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 406.0,
|
|
"completions/max_terminated_length": 406.0,
|
|
"completions/mean_length": 186.525,
|
|
"completions/mean_terminated_length": 186.525,
|
|
"completions/min_length": 70.7,
|
|
"completions/min_terminated_length": 70.7,
|
|
"entropy": 0.43793474808335303,
|
|
"epoch": 0.1734258271077908,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.02989210784435272,
|
|
"num_tokens": 6095054.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.22390617728233336,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.22390617728233336,
|
|
"step": 1300,
|
|
"step_time": 9.779177985247225,
|
|
"zero_adv_skip_rate": 0.3892
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 408.2,
|
|
"completions/max_terminated_length": 264.7,
|
|
"completions/mean_length": 167.625,
|
|
"completions/mean_terminated_length": 147.16785888671876,
|
|
"completions/min_length": 62.8,
|
|
"completions/min_terminated_length": 62.8,
|
|
"entropy": 0.4277988678465287,
|
|
"epoch": 0.1747598719316969,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.043900468945503236,
|
|
"num_tokens": 6142976.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.26849985122680664,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.2684998571872711,
|
|
"step": 1310,
|
|
"step_time": 10.056305562797935,
|
|
"zero_adv_skip_rate": 0.3893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 673.8,
|
|
"completions/max_terminated_length": 448.7,
|
|
"completions/mean_length": 237.35,
|
|
"completions/mean_terminated_length": 194.74286346435548,
|
|
"completions/min_length": 82.7,
|
|
"completions/min_terminated_length": 82.7,
|
|
"entropy": 0.43072928790934384,
|
|
"epoch": 0.17609391675560299,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.64453125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.03903071582317352,
|
|
"num_tokens": 6190380.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.3501461952924728,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.35014620125293733,
|
|
"step": 1320,
|
|
"step_time": 16.139569891989233,
|
|
"zero_adv_skip_rate": 0.3879
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 419.9,
|
|
"completions/max_terminated_length": 246.6,
|
|
"completions/mean_length": 154.5125,
|
|
"completions/mean_terminated_length": 130.97321472167968,
|
|
"completions/min_length": 67.6,
|
|
"completions/min_terminated_length": 67.6,
|
|
"entropy": 0.4000225051173142,
|
|
"epoch": 0.17742796157950907,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.52734375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.01474658101797104,
|
|
"num_tokens": 6244613.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.3294931024312973,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.32949312031269073,
|
|
"step": 1330,
|
|
"step_time": 10.394404539186507,
|
|
"zero_adv_skip_rate": 0.3872
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 717.9,
|
|
"completions/max_terminated_length": 536.8,
|
|
"completions/mean_length": 258.575,
|
|
"completions/mean_terminated_length": 219.04108047485352,
|
|
"completions/min_length": 92.8,
|
|
"completions/min_terminated_length": 92.8,
|
|
"entropy": 0.39575998360912007,
|
|
"epoch": 0.17876200640341516,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.84765625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.009406913816928864,
|
|
"num_tokens": 6293331.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.2739637613296509,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.27396377325057986,
|
|
"step": 1340,
|
|
"step_time": 16.70131459068507,
|
|
"zero_adv_skip_rate": 0.3873
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 305.1,
|
|
"completions/max_terminated_length": 274.5,
|
|
"completions/mean_length": 157.325,
|
|
"completions/mean_terminated_length": 150.3839309692383,
|
|
"completions/min_length": 89.1,
|
|
"completions/min_terminated_length": 89.1,
|
|
"entropy": 0.4429144486784935,
|
|
"epoch": 0.18009605122732122,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.65625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.003510556370019913,
|
|
"num_tokens": 6335533.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.08711025416851044,
|
|
"rewards/correctness_reward/mean": 0.175,
|
|
"rewards/correctness_reward/std": 0.08711026012897491,
|
|
"step": 1350,
|
|
"step_time": 7.659681435395032,
|
|
"zero_adv_skip_rate": 0.3904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 766.4,
|
|
"completions/max_terminated_length": 451.9,
|
|
"completions/mean_length": 286.0375,
|
|
"completions/mean_terminated_length": 226.44464721679688,
|
|
"completions/min_length": 102.3,
|
|
"completions/min_terminated_length": 102.3,
|
|
"entropy": 0.49388060681521895,
|
|
"epoch": 0.1814300960512273,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.006638549268245697,
|
|
"num_tokens": 6386376.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.2512414216995239,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.2512414276599884,
|
|
"step": 1360,
|
|
"step_time": 18.118536785524338,
|
|
"zero_adv_skip_rate": 0.3912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 583.6,
|
|
"completions/max_terminated_length": 459.3,
|
|
"completions/mean_length": 242.4,
|
|
"completions/mean_terminated_length": 221.03928909301757,
|
|
"completions/min_length": 99.4,
|
|
"completions/min_terminated_length": 99.4,
|
|
"entropy": 0.34554270728092107,
|
|
"epoch": 0.1827641408751334,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.005892095342278481,
|
|
"num_tokens": 6434856.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.34042876958847046,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.3404287874698639,
|
|
"step": 1370,
|
|
"step_time": 14.254161248356104,
|
|
"zero_adv_skip_rate": 0.3905
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 598.0,
|
|
"completions/max_terminated_length": 438.5,
|
|
"completions/mean_length": 190.6875,
|
|
"completions/mean_terminated_length": 166.63571472167968,
|
|
"completions/min_length": 82.0,
|
|
"completions/min_terminated_length": 82.0,
|
|
"entropy": 0.44241910881828517,
|
|
"epoch": 0.1840981856990395,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.024475905299186706,
|
|
"num_tokens": 6480431.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.36484059393405915,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.36484062373638154,
|
|
"step": 1380,
|
|
"step_time": 14.39565998762846,
|
|
"zero_adv_skip_rate": 0.3891
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 433.7,
|
|
"completions/max_terminated_length": 395.9,
|
|
"completions/mean_length": 178.4875,
|
|
"completions/mean_terminated_length": 171.7946449279785,
|
|
"completions/min_length": 82.2,
|
|
"completions/min_terminated_length": 82.2,
|
|
"entropy": 0.48214011564850806,
|
|
"epoch": 0.18543223052294558,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.032677295804023745,
|
|
"num_tokens": 6521478.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.3375,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 1390,
|
|
"step_time": 11.051581164076925,
|
|
"zero_adv_skip_rate": 0.3899
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 469.8,
|
|
"completions/max_terminated_length": 271.9,
|
|
"completions/mean_length": 164.925,
|
|
"completions/mean_terminated_length": 139.1982162475586,
|
|
"completions/min_length": 74.8,
|
|
"completions/min_terminated_length": 74.8,
|
|
"entropy": 0.39956175070255995,
|
|
"epoch": 0.18676627534685167,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.69921875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.14255974292755128,
|
|
"num_tokens": 6563752.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.3737070143222809,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.37370702624320984,
|
|
"step": 1400,
|
|
"step_time": 11.805594439059496,
|
|
"zero_adv_skip_rate": 0.3886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 537.8,
|
|
"completions/max_terminated_length": 380.8,
|
|
"completions/mean_length": 229.375,
|
|
"completions/mean_terminated_length": 204.10714569091797,
|
|
"completions/min_length": 115.0,
|
|
"completions/min_terminated_length": 115.0,
|
|
"entropy": 0.2976266074304779,
|
|
"epoch": 0.18810032017075773,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.66015625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0157033234834671,
|
|
"num_tokens": 6609958.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.3084602475166321,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.3084602653980255,
|
|
"step": 1410,
|
|
"step_time": 12.948151403665543,
|
|
"zero_adv_skip_rate": 0.3887
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 507.5,
|
|
"completions/max_terminated_length": 363.2,
|
|
"completions/mean_length": 246.9125,
|
|
"completions/mean_terminated_length": 224.1678596496582,
|
|
"completions/min_length": 105.9,
|
|
"completions/min_terminated_length": 105.9,
|
|
"entropy": 0.39080538460984826,
|
|
"epoch": 0.18943436499466382,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.013028408586978912,
|
|
"num_tokens": 6659095.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.18855083882808685,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.18855083882808685,
|
|
"step": 1420,
|
|
"step_time": 12.166908160224557,
|
|
"zero_adv_skip_rate": 0.3901
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 629.7,
|
|
"completions/max_terminated_length": 629.7,
|
|
"completions/mean_length": 221.1375,
|
|
"completions/mean_terminated_length": 221.1375,
|
|
"completions/min_length": 84.8,
|
|
"completions/min_terminated_length": 84.8,
|
|
"entropy": 0.4593142982572317,
|
|
"epoch": 0.1907684098185699,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.63671875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.019391410052776337,
|
|
"num_tokens": 6704826.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.22051151394844054,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.2205115258693695,
|
|
"step": 1430,
|
|
"step_time": 14.70087132724002,
|
|
"zero_adv_skip_rate": 0.3909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 483.7,
|
|
"completions/max_terminated_length": 483.7,
|
|
"completions/mean_length": 216.4375,
|
|
"completions/mean_terminated_length": 216.4375,
|
|
"completions/min_length": 87.8,
|
|
"completions/min_terminated_length": 87.8,
|
|
"entropy": 0.4447890114194403,
|
|
"epoch": 0.192102454642476,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.60546875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0009842038154602051,
|
|
"num_tokens": 6757509.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.2777381867170334,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.27773819863796234,
|
|
"step": 1440,
|
|
"step_time": 11.723635268304497,
|
|
"zero_adv_skip_rate": 0.391
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 412.5,
|
|
"completions/max_terminated_length": 411.7,
|
|
"completions/mean_length": 196.3625,
|
|
"completions/mean_terminated_length": 195.63214340209962,
|
|
"completions/min_length": 90.5,
|
|
"completions/min_terminated_length": 90.5,
|
|
"entropy": 0.37465089639382704,
|
|
"epoch": 0.19343649946638208,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.08714512586593628,
|
|
"num_tokens": 6802890.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.331182587146759,
|
|
"rewards/correctness_reward/mean": 0.225,
|
|
"rewards/correctness_reward/std": 0.33118261098861695,
|
|
"step": 1450,
|
|
"step_time": 10.698044626321643,
|
|
"zero_adv_skip_rate": 0.3903
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 401.1,
|
|
"completions/max_terminated_length": 399.6,
|
|
"completions/mean_length": 174.25,
|
|
"completions/mean_terminated_length": 173.9125,
|
|
"completions/min_length": 64.2,
|
|
"completions/min_terminated_length": 64.2,
|
|
"entropy": 0.36882760893139577,
|
|
"epoch": 0.19477054429028814,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.1953125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.01814217120409012,
|
|
"num_tokens": 6845470.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.44017202258110044,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.4401720404624939,
|
|
"step": 1460,
|
|
"step_time": 10.522196082677691,
|
|
"zero_adv_skip_rate": 0.3884
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 273.8,
|
|
"completions/max_terminated_length": 235.3,
|
|
"completions/mean_length": 135.325,
|
|
"completions/mean_terminated_length": 129.47857360839845,
|
|
"completions/min_length": 77.6,
|
|
"completions/min_terminated_length": 77.6,
|
|
"entropy": 0.5287603926844895,
|
|
"epoch": 0.19610458911419423,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 1.2890625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.018582789599895476,
|
|
"num_tokens": 6888072.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.18855083882808685,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.18855083882808685,
|
|
"step": 1470,
|
|
"step_time": 7.1139101113192735,
|
|
"zero_adv_skip_rate": 0.3898
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 451.8,
|
|
"completions/max_terminated_length": 441.5,
|
|
"completions/mean_length": 237.375,
|
|
"completions/mean_terminated_length": 228.90714416503906,
|
|
"completions/min_length": 116.0,
|
|
"completions/min_terminated_length": 116.0,
|
|
"entropy": 0.5181247236207127,
|
|
"epoch": 0.19743863393810032,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04910776615142822,
|
|
"num_tokens": 6937310.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.25331066846847533,
|
|
"rewards/correctness_reward/mean": 0.175,
|
|
"rewards/correctness_reward/std": 0.25331069231033326,
|
|
"step": 1480,
|
|
"step_time": 11.06982277398929,
|
|
"zero_adv_skip_rate": 0.3905
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 500.5,
|
|
"completions/max_terminated_length": 499.5,
|
|
"completions/mean_length": 269.5625,
|
|
"completions/mean_terminated_length": 268.9392860412598,
|
|
"completions/min_length": 109.7,
|
|
"completions/min_terminated_length": 109.7,
|
|
"entropy": 0.4296047078445554,
|
|
"epoch": 0.1987726787620064,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.055455726385116574,
|
|
"num_tokens": 6988787.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.24200308620929717,
|
|
"rewards/correctness_reward/mean": 0.1875,
|
|
"rewards/correctness_reward/std": 0.24200308620929717,
|
|
"step": 1490,
|
|
"step_time": 11.976779755204916,
|
|
"zero_adv_skip_rate": 0.3913
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 434.5,
|
|
"completions/max_terminated_length": 434.4,
|
|
"completions/mean_length": 227.5625,
|
|
"completions/mean_terminated_length": 227.525,
|
|
"completions/min_length": 101.4,
|
|
"completions/min_terminated_length": 101.4,
|
|
"entropy": 0.3801997581496835,
|
|
"epoch": 0.2001067235859125,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.057369178533554076,
|
|
"num_tokens": 7036008.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.23144718110561371,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.23144719302654265,
|
|
"step": 1500,
|
|
"step_time": 10.599012072198093,
|
|
"zero_adv_skip_rate": 0.392
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 949.5,
|
|
"completions/max_terminated_length": 465.0,
|
|
"completions/mean_length": 271.7375,
|
|
"completions/mean_terminated_length": 202.8464324951172,
|
|
"completions/min_length": 86.1,
|
|
"completions/min_terminated_length": 86.1,
|
|
"entropy": 0.4548583905212581,
|
|
"epoch": 0.20144076840981856,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 1.09375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.07914146780967712,
|
|
"num_tokens": 7086227.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.24577751159667968,
|
|
"rewards/correctness_reward/mean": 0.225,
|
|
"rewards/correctness_reward/std": 0.24577751159667968,
|
|
"step": 1510,
|
|
"step_time": 22.064335219562054,
|
|
"zero_adv_skip_rate": 0.3927
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 712.3,
|
|
"completions/max_terminated_length": 383.5,
|
|
"completions/mean_length": 266.75,
|
|
"completions/mean_terminated_length": 220.66428985595704,
|
|
"completions/min_length": 95.8,
|
|
"completions/min_terminated_length": 95.8,
|
|
"entropy": 0.443096111373355,
|
|
"epoch": 0.20277481323372465,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.255859375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.019948188960552216,
|
|
"num_tokens": 7135607.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.24662851691246032,
|
|
"rewards/correctness_reward/mean": 0.2125,
|
|
"rewards/correctness_reward/std": 0.2466285228729248,
|
|
"step": 1520,
|
|
"step_time": 17.176886401791126,
|
|
"zero_adv_skip_rate": 0.3928
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 735.4,
|
|
"completions/max_terminated_length": 262.0,
|
|
"completions/mean_length": 222.1875,
|
|
"completions/mean_terminated_length": 154.46964416503906,
|
|
"completions/min_length": 72.2,
|
|
"completions/min_terminated_length": 72.2,
|
|
"entropy": 0.41681125946342945,
|
|
"epoch": 0.20410885805763074,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.66015625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.009188344329595565,
|
|
"num_tokens": 7180238.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.1940147489309311,
|
|
"rewards/correctness_reward/mean": 0.45,
|
|
"rewards/correctness_reward/std": 0.19401475489139558,
|
|
"step": 1530,
|
|
"step_time": 17.516076264623553,
|
|
"zero_adv_skip_rate": 0.3941
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 526.0,
|
|
"completions/max_terminated_length": 325.7,
|
|
"completions/mean_length": 172.45,
|
|
"completions/mean_terminated_length": 143.47321701049805,
|
|
"completions/min_length": 69.6,
|
|
"completions/min_terminated_length": 69.6,
|
|
"entropy": 0.4738533428559701,
|
|
"epoch": 0.20544290288153683,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 1.0078125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.047610771656036374,
|
|
"num_tokens": 7223234.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.26680251955986023,
|
|
"rewards/correctness_reward/mean": 0.225,
|
|
"rewards/correctness_reward/std": 0.2668025314807892,
|
|
"step": 1540,
|
|
"step_time": 12.946707313787192,
|
|
"zero_adv_skip_rate": 0.3942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 427.9,
|
|
"completions/max_terminated_length": 240.5,
|
|
"completions/mean_length": 173.85,
|
|
"completions/mean_terminated_length": 145.2392906188965,
|
|
"completions/min_length": 75.1,
|
|
"completions/min_terminated_length": 75.1,
|
|
"entropy": 0.4072856716811657,
|
|
"epoch": 0.20677694770544292,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.62109375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04574887156486511,
|
|
"num_tokens": 7268526.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.26133076250553133,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.26133078038692475,
|
|
"step": 1550,
|
|
"step_time": 10.996624306589364,
|
|
"zero_adv_skip_rate": 0.3942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 332.8,
|
|
"completions/max_terminated_length": 317.5,
|
|
"completions/mean_length": 162.05,
|
|
"completions/mean_terminated_length": 153.8303596496582,
|
|
"completions/min_length": 71.1,
|
|
"completions/min_terminated_length": 71.1,
|
|
"entropy": 0.3999555390328169,
|
|
"epoch": 0.20811099252934898,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.002561773732304573,
|
|
"num_tokens": 7314002.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.3130935251712799,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.3130935370922089,
|
|
"step": 1560,
|
|
"step_time": 9.0249317525886,
|
|
"zero_adv_skip_rate": 0.3936
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 462.5,
|
|
"completions/max_terminated_length": 319.2,
|
|
"completions/mean_length": 219.95,
|
|
"completions/mean_terminated_length": 190.15714912414552,
|
|
"completions/min_length": 86.3,
|
|
"completions/min_terminated_length": 86.3,
|
|
"entropy": 0.3465527434600517,
|
|
"epoch": 0.20944503735325506,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.21875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0006245911121368408,
|
|
"num_tokens": 7361438.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.3337466180324554,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.3337466180324554,
|
|
"step": 1570,
|
|
"step_time": 12.064378005266189,
|
|
"zero_adv_skip_rate": 0.3924
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 751.3,
|
|
"completions/max_terminated_length": 460.2,
|
|
"completions/mean_length": 235.975,
|
|
"completions/mean_terminated_length": 193.9892890930176,
|
|
"completions/min_length": 73.4,
|
|
"completions/min_terminated_length": 73.4,
|
|
"entropy": 0.3856712769096096,
|
|
"epoch": 0.21077908217716115,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.015856002271175385,
|
|
"num_tokens": 7407836.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.24493903219699859,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.24493903219699859,
|
|
"step": 1580,
|
|
"step_time": 18.039961303770543,
|
|
"zero_adv_skip_rate": 0.3924
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 532.3,
|
|
"completions/max_terminated_length": 529.7,
|
|
"completions/mean_length": 227.375,
|
|
"completions/mean_terminated_length": 226.63035736083984,
|
|
"completions/min_length": 76.9,
|
|
"completions/min_terminated_length": 76.9,
|
|
"entropy": 0.3544897949323058,
|
|
"epoch": 0.21211312700106724,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.94140625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.00663042813539505,
|
|
"num_tokens": 7452890.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.26302809417247774,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.26302810609340666,
|
|
"step": 1590,
|
|
"step_time": 13.146477149892599,
|
|
"zero_adv_skip_rate": 0.3925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 305.9,
|
|
"completions/max_terminated_length": 295.1,
|
|
"completions/mean_length": 168.0125,
|
|
"completions/mean_terminated_length": 165.2607162475586,
|
|
"completions/min_length": 82.4,
|
|
"completions/min_terminated_length": 82.4,
|
|
"entropy": 0.39410924948751924,
|
|
"epoch": 0.21344717182497333,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.56640625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.017206950485706328,
|
|
"num_tokens": 7496491.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.20580926835536956,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.20580926835536956,
|
|
"step": 1600,
|
|
"step_time": 8.189897743798792,
|
|
"zero_adv_skip_rate": 0.3931
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 441.0,
|
|
"completions/max_terminated_length": 436.5,
|
|
"completions/mean_length": 212.3375,
|
|
"completions/mean_terminated_length": 211.18750076293946,
|
|
"completions/min_length": 77.3,
|
|
"completions/min_terminated_length": 77.3,
|
|
"entropy": 0.49083068687468767,
|
|
"epoch": 0.2147812166488794,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.1356847405433655,
|
|
"num_tokens": 7540726.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.25210027396678925,
|
|
"rewards/correctness_reward/mean": 0.2375,
|
|
"rewards/correctness_reward/std": 0.25210027396678925,
|
|
"step": 1610,
|
|
"step_time": 11.280258295219392,
|
|
"zero_adv_skip_rate": 0.3932
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 806.7,
|
|
"completions/max_terminated_length": 635.9,
|
|
"completions/mean_length": 279.025,
|
|
"completions/mean_terminated_length": 254.70893173217775,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"entropy": 0.37408145357455524,
|
|
"epoch": 0.21611526147278548,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.453125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.10189423561096192,
|
|
"num_tokens": 7592648.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.29838343262672423,
|
|
"rewards/correctness_reward/mean": 0.2,
|
|
"rewards/correctness_reward/std": 0.2983834445476532,
|
|
"step": 1620,
|
|
"step_time": 19.43022430445999,
|
|
"zero_adv_skip_rate": 0.3926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 378.4,
|
|
"completions/max_terminated_length": 356.9,
|
|
"completions/mean_length": 201.4875,
|
|
"completions/mean_terminated_length": 197.31964416503905,
|
|
"completions/min_length": 82.1,
|
|
"completions/min_terminated_length": 82.1,
|
|
"entropy": 0.3806605231948197,
|
|
"epoch": 0.21744930629669157,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04960947632789612,
|
|
"num_tokens": 7636151.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.2739637613296509,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.27396377325057986,
|
|
"step": 1630,
|
|
"step_time": 9.653176853898913,
|
|
"zero_adv_skip_rate": 0.3926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 189.4,
|
|
"completions/max_terminated_length": 182.8,
|
|
"completions/mean_length": 105.8375,
|
|
"completions/mean_terminated_length": 104.03392868041992,
|
|
"completions/min_length": 65.1,
|
|
"completions/min_terminated_length": 65.1,
|
|
"entropy": 0.4038819247590644,
|
|
"epoch": 0.21878335112059766,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.7265625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.054366469383239746,
|
|
"num_tokens": 7671930.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.3366621911525726,
|
|
"rewards/correctness_reward/mean": 0.4375,
|
|
"rewards/correctness_reward/std": 0.33666219711303713,
|
|
"step": 1640,
|
|
"step_time": 5.7509320172481235,
|
|
"zero_adv_skip_rate": 0.3921
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 303.4,
|
|
"completions/max_terminated_length": 285.6,
|
|
"completions/mean_length": 159.625,
|
|
"completions/mean_terminated_length": 154.5857162475586,
|
|
"completions/min_length": 60.7,
|
|
"completions/min_terminated_length": 60.7,
|
|
"entropy": 0.38768250844441354,
|
|
"epoch": 0.22011739594450375,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.55078125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.10913441181182862,
|
|
"num_tokens": 7715484.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.3593766838312149,
|
|
"rewards/correctness_reward/mean": 0.4375,
|
|
"rewards/correctness_reward/std": 0.35937670767307284,
|
|
"step": 1650,
|
|
"step_time": 8.55328368442133,
|
|
"zero_adv_skip_rate": 0.3909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 755.0,
|
|
"completions/max_terminated_length": 420.9,
|
|
"completions/mean_length": 234.4,
|
|
"completions/mean_terminated_length": 188.466072845459,
|
|
"completions/min_length": 63.3,
|
|
"completions/min_terminated_length": 63.3,
|
|
"entropy": 0.4034078694466088,
|
|
"epoch": 0.2214514407684098,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.5234375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.12172737121582031,
|
|
"num_tokens": 7761548.0,
|
|
"reward": 0.5125,
|
|
"reward_std": 0.4090623527765274,
|
|
"rewards/correctness_reward/mean": 0.5125,
|
|
"rewards/correctness_reward/std": 0.4090623646974564,
|
|
"step": 1660,
|
|
"step_time": 18.620233062841,
|
|
"zero_adv_skip_rate": 0.3892
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 982.1,
|
|
"completions/max_terminated_length": 484.8,
|
|
"completions/mean_length": 296.4375,
|
|
"completions/mean_terminated_length": 225.74107360839844,
|
|
"completions/min_length": 85.7,
|
|
"completions/min_terminated_length": 85.7,
|
|
"entropy": 0.35793439447879793,
|
|
"epoch": 0.2227854855923159,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.07906410694122315,
|
|
"num_tokens": 7813567.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.22220884561538695,
|
|
"rewards/correctness_reward/mean": 0.1375,
|
|
"rewards/correctness_reward/std": 0.22220885157585143,
|
|
"step": 1670,
|
|
"step_time": 22.751673092320562,
|
|
"zero_adv_skip_rate": 0.3898
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 855.0,
|
|
"completions/max_terminated_length": 716.7,
|
|
"completions/mean_length": 282.8625,
|
|
"completions/mean_terminated_length": 261.69464416503905,
|
|
"completions/min_length": 107.8,
|
|
"completions/min_terminated_length": 107.8,
|
|
"entropy": 0.2337462310679257,
|
|
"epoch": 0.22411953041622198,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0010165367275476456,
|
|
"num_tokens": 7863940.0,
|
|
"reward": 0.475,
|
|
"reward_std": 0.19062008559703827,
|
|
"rewards/correctness_reward/mean": 0.475,
|
|
"rewards/correctness_reward/std": 0.1906201034784317,
|
|
"step": 1680,
|
|
"step_time": 19.92497768290341,
|
|
"zero_adv_skip_rate": 0.3911
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 667.8,
|
|
"completions/max_terminated_length": 489.3,
|
|
"completions/mean_length": 247.7625,
|
|
"completions/mean_terminated_length": 220.55893325805664,
|
|
"completions/min_length": 107.3,
|
|
"completions/min_terminated_length": 107.3,
|
|
"entropy": 0.47232110565528274,
|
|
"epoch": 0.22545357524012807,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.021039922535419465,
|
|
"num_tokens": 7913129.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.16875659823417663,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.1687566041946411,
|
|
"step": 1690,
|
|
"step_time": 16.056975765991957,
|
|
"zero_adv_skip_rate": 0.3923
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 605.3,
|
|
"completions/max_terminated_length": 390.2,
|
|
"completions/mean_length": 214.475,
|
|
"completions/mean_terminated_length": 184.55178756713866,
|
|
"completions/min_length": 90.1,
|
|
"completions/min_terminated_length": 90.1,
|
|
"entropy": 0.532163928821683,
|
|
"epoch": 0.22678762006403416,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.8359375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.05198777318000793,
|
|
"num_tokens": 7959503.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.24408017992973327,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.24408018589019775,
|
|
"step": 1700,
|
|
"step_time": 14.874868696182967,
|
|
"zero_adv_skip_rate": 0.3929
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 402.9,
|
|
"completions/max_terminated_length": 357.6,
|
|
"completions/mean_length": 193.825,
|
|
"completions/mean_terminated_length": 186.05892944335938,
|
|
"completions/min_length": 78.3,
|
|
"completions/min_terminated_length": 78.3,
|
|
"entropy": 0.39755881934737164,
|
|
"epoch": 0.22812166488794022,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.14553756713867189,
|
|
"num_tokens": 8006065.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.2504029482603073,
|
|
"step": 1710,
|
|
"step_time": 10.387977225519716,
|
|
"zero_adv_skip_rate": 0.393
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 431.7,
|
|
"completions/max_terminated_length": 263.5,
|
|
"completions/mean_length": 183.675,
|
|
"completions/mean_terminated_length": 160.49643096923828,
|
|
"completions/min_length": 76.0,
|
|
"completions/min_terminated_length": 76.0,
|
|
"entropy": 0.3991669893730432,
|
|
"epoch": 0.2294557097118463,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0057064443826675415,
|
|
"num_tokens": 8052063.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.33751319646835326,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.3375132083892822,
|
|
"step": 1720,
|
|
"step_time": 11.455593496654183,
|
|
"zero_adv_skip_rate": 0.3919
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 462.7,
|
|
"completions/max_terminated_length": 291.6,
|
|
"completions/mean_length": 172.275,
|
|
"completions/mean_terminated_length": 148.31964302062988,
|
|
"completions/min_length": 66.4,
|
|
"completions/min_terminated_length": 66.4,
|
|
"entropy": 0.40418651346117257,
|
|
"epoch": 0.2307897545357524,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0019517064094543457,
|
|
"num_tokens": 8094797.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.18771235942840575,
|
|
"rewards/correctness_reward/mean": 0.225,
|
|
"rewards/correctness_reward/std": 0.18771235942840575,
|
|
"step": 1730,
|
|
"step_time": 11.707576542627066,
|
|
"zero_adv_skip_rate": 0.3925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 379.6,
|
|
"completions/max_terminated_length": 372.3,
|
|
"completions/mean_length": 170.5125,
|
|
"completions/mean_terminated_length": 167.14464569091797,
|
|
"completions/min_length": 87.6,
|
|
"completions/min_terminated_length": 87.6,
|
|
"entropy": 0.30867259681690484,
|
|
"epoch": 0.2321237993596585,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.765625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.15261980295181274,
|
|
"num_tokens": 8135238.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.3409078598022461,
|
|
"rewards/correctness_reward/mean": 0.45,
|
|
"rewards/correctness_reward/std": 0.3409078598022461,
|
|
"step": 1740,
|
|
"step_time": 10.301522635947913,
|
|
"zero_adv_skip_rate": 0.3914
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 996.6,
|
|
"completions/max_terminated_length": 699.5,
|
|
"completions/mean_length": 334.3875,
|
|
"completions/mean_terminated_length": 258.11548233032227,
|
|
"completions/min_length": 80.9,
|
|
"completions/min_terminated_length": 80.9,
|
|
"entropy": 0.3342395467417581,
|
|
"epoch": 0.23345784418356458,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1371429443359375,
|
|
"num_tokens": 8189333.0,
|
|
"reward": 0.525,
|
|
"reward_std": 0.2966939479112625,
|
|
"rewards/correctness_reward/mean": 0.525,
|
|
"rewards/correctness_reward/std": 0.296693953871727,
|
|
"step": 1750,
|
|
"step_time": 23.852908514998852,
|
|
"zero_adv_skip_rate": 0.3909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 479.1,
|
|
"completions/max_terminated_length": 307.8,
|
|
"completions/mean_length": 179.0875,
|
|
"completions/mean_terminated_length": 155.14285736083986,
|
|
"completions/min_length": 65.0,
|
|
"completions/min_terminated_length": 65.0,
|
|
"entropy": 0.3390505961807711,
|
|
"epoch": 0.23479188900747064,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.6171875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.047684741020202634,
|
|
"num_tokens": 8233252.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.3257186770439148,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.32571869492530825,
|
|
"step": 1760,
|
|
"step_time": 12.497120458073914,
|
|
"zero_adv_skip_rate": 0.3903
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 617.6,
|
|
"completions/max_terminated_length": 420.4,
|
|
"completions/mean_length": 193.4125,
|
|
"completions/mean_terminated_length": 167.8428581237793,
|
|
"completions/min_length": 75.5,
|
|
"completions/min_terminated_length": 75.5,
|
|
"entropy": 0.33957797350982827,
|
|
"epoch": 0.23612593383137673,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.73828125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1463964104652405,
|
|
"num_tokens": 8278149.0,
|
|
"reward": 0.4875,
|
|
"reward_std": 0.261338609457016,
|
|
"rewards/correctness_reward/mean": 0.4875,
|
|
"rewards/correctness_reward/std": 0.2613386154174805,
|
|
"step": 1770,
|
|
"step_time": 15.577880680188537,
|
|
"zero_adv_skip_rate": 0.3904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 272.7,
|
|
"completions/max_terminated_length": 256.9,
|
|
"completions/mean_length": 164.5375,
|
|
"completions/mean_terminated_length": 161.43035812377929,
|
|
"completions/min_length": 82.2,
|
|
"completions/min_terminated_length": 82.2,
|
|
"entropy": 0.3665262874215841,
|
|
"epoch": 0.23745997865528282,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 1.0546875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.022800447046756746,
|
|
"num_tokens": 8320456.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.20411193668842315,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.20411194264888763,
|
|
"step": 1780,
|
|
"step_time": 7.533414826635271,
|
|
"zero_adv_skip_rate": 0.391
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 867.8,
|
|
"completions/max_terminated_length": 742.3,
|
|
"completions/mean_length": 270.3,
|
|
"completions/mean_terminated_length": 247.83393096923828,
|
|
"completions/min_length": 85.4,
|
|
"completions/min_terminated_length": 85.4,
|
|
"entropy": 0.3625253204566737,
|
|
"epoch": 0.2387940234791889,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.341796875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.08573217391967773,
|
|
"num_tokens": 8374616.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.26302809417247774,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.26302810609340666,
|
|
"step": 1790,
|
|
"step_time": 21.003793887700887,
|
|
"zero_adv_skip_rate": 0.3911
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 744.1,
|
|
"completions/max_terminated_length": 301.2,
|
|
"completions/mean_length": 207.275,
|
|
"completions/mean_terminated_length": 138.12143249511718,
|
|
"completions/min_length": 75.7,
|
|
"completions/min_terminated_length": 75.7,
|
|
"entropy": 0.3443771672124664,
|
|
"epoch": 0.240128068303095,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.3828125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.11030049324035644,
|
|
"num_tokens": 8417470.0,
|
|
"reward": 0.4875,
|
|
"reward_std": 0.2558668524026871,
|
|
"rewards/correctness_reward/mean": 0.4875,
|
|
"rewards/correctness_reward/std": 0.25586686432361605,
|
|
"step": 1800,
|
|
"step_time": 18.316201197169722,
|
|
"zero_adv_skip_rate": 0.3911
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 341.1,
|
|
"completions/max_terminated_length": 339.1,
|
|
"completions/mean_length": 157.1875,
|
|
"completions/mean_terminated_length": 155.57857208251954,
|
|
"completions/min_length": 65.2,
|
|
"completions/min_terminated_length": 65.2,
|
|
"entropy": 0.4255388995100345,
|
|
"epoch": 0.24146211312700105,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0007633522152900695,
|
|
"num_tokens": 8460453.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.3130935251712799,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.3130935370922089,
|
|
"step": 1810,
|
|
"step_time": 9.512513293512166,
|
|
"zero_adv_skip_rate": 0.3906
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 451.6,
|
|
"completions/max_terminated_length": 300.5,
|
|
"completions/mean_length": 181.0125,
|
|
"completions/mean_terminated_length": 157.6482147216797,
|
|
"completions/min_length": 65.6,
|
|
"completions/min_terminated_length": 65.6,
|
|
"entropy": 0.38209283351898193,
|
|
"epoch": 0.24279615795090714,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.185546875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.044084632396698,
|
|
"num_tokens": 8502630.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.19864802658557892,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.19864802658557892,
|
|
"step": 1820,
|
|
"step_time": 11.56212072521448,
|
|
"zero_adv_skip_rate": 0.3912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 381.0,
|
|
"completions/max_terminated_length": 375.3,
|
|
"completions/mean_length": 170.9,
|
|
"completions/mean_terminated_length": 169.75535736083984,
|
|
"completions/min_length": 71.9,
|
|
"completions/min_terminated_length": 71.9,
|
|
"entropy": 0.3688055775128305,
|
|
"epoch": 0.24413020277481323,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.412109375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.11165738105773926,
|
|
"num_tokens": 8543742.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.3265775293111801,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.3265775412321091,
|
|
"step": 1830,
|
|
"step_time": 10.322372621949762,
|
|
"zero_adv_skip_rate": 0.3902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 710.6,
|
|
"completions/max_terminated_length": 540.5,
|
|
"completions/mean_length": 233.3625,
|
|
"completions/mean_terminated_length": 190.96607551574706,
|
|
"completions/min_length": 79.3,
|
|
"completions/min_terminated_length": 79.3,
|
|
"entropy": 0.42768936417996883,
|
|
"epoch": 0.24546424759871932,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.039540189504623416,
|
|
"num_tokens": 8589923.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.21674493551254273,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.21674493551254273,
|
|
"step": 1840,
|
|
"step_time": 17.450603359844536,
|
|
"zero_adv_skip_rate": 0.3908
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 395.2,
|
|
"completions/max_terminated_length": 388.8,
|
|
"completions/mean_length": 176.4375,
|
|
"completions/mean_terminated_length": 175.11607208251954,
|
|
"completions/min_length": 83.1,
|
|
"completions/min_terminated_length": 83.1,
|
|
"entropy": 0.3433068534359336,
|
|
"epoch": 0.2467982924226254,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.6796875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.020267611742019652,
|
|
"num_tokens": 8632190.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.2722664296627045,
|
|
"rewards/correctness_reward/mean": 0.4375,
|
|
"rewards/correctness_reward/std": 0.2722664475440979,
|
|
"step": 1850,
|
|
"step_time": 10.475295317918063,
|
|
"zero_adv_skip_rate": 0.3908
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 604.8,
|
|
"completions/max_terminated_length": 462.1,
|
|
"completions/mean_length": 213.8375,
|
|
"completions/mean_terminated_length": 191.8625015258789,
|
|
"completions/min_length": 77.7,
|
|
"completions/min_terminated_length": 77.7,
|
|
"entropy": 0.3395872559132321,
|
|
"epoch": 0.24813233724653147,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.018883098661899567,
|
|
"num_tokens": 8678361.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.3475900113582611,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.34759002923965454,
|
|
"step": 1860,
|
|
"step_time": 15.5262266933918,
|
|
"zero_adv_skip_rate": 0.3903
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 611.6,
|
|
"completions/max_terminated_length": 282.8,
|
|
"completions/mean_length": 198.325,
|
|
"completions/mean_terminated_length": 152.11786193847655,
|
|
"completions/min_length": 70.1,
|
|
"completions/min_terminated_length": 70.1,
|
|
"entropy": 0.3438689954346046,
|
|
"epoch": 0.24946638207043756,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.671875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.13673205375671388,
|
|
"num_tokens": 8721411.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.3484488636255264,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.34844887554645537,
|
|
"step": 1870,
|
|
"step_time": 15.425224483851343,
|
|
"zero_adv_skip_rate": 0.3893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 252.0,
|
|
"completions/max_terminated_length": 252.0,
|
|
"completions/mean_length": 122.3125,
|
|
"completions/mean_terminated_length": 122.3125,
|
|
"completions/min_length": 47.4,
|
|
"completions/min_terminated_length": 47.4,
|
|
"entropy": 0.5004538457308497,
|
|
"epoch": 0.2508004268943437,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.76171875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.02021917700767517,
|
|
"num_tokens": 8758388.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.31310137212276457,
|
|
"rewards/correctness_reward/mean": 0.4375,
|
|
"rewards/correctness_reward/std": 0.31310137212276457,
|
|
"step": 1880,
|
|
"step_time": 7.212355415988713,
|
|
"zero_adv_skip_rate": 0.3888
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 309.5,
|
|
"completions/max_terminated_length": 299.8,
|
|
"completions/mean_length": 155.85,
|
|
"completions/mean_terminated_length": 154.05535812377929,
|
|
"completions/min_length": 75.8,
|
|
"completions/min_terminated_length": 75.8,
|
|
"entropy": 0.4292975179851055,
|
|
"epoch": 0.2521344717182497,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.03515174388885498,
|
|
"num_tokens": 8800064.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.2886660069227219,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.28866603076457975,
|
|
"step": 1890,
|
|
"step_time": 8.586889522429555,
|
|
"zero_adv_skip_rate": 0.3889
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 649.3,
|
|
"completions/max_terminated_length": 560.0,
|
|
"completions/mean_length": 259.7625,
|
|
"completions/mean_terminated_length": 213.55178604125976,
|
|
"completions/min_length": 81.1,
|
|
"completions/min_terminated_length": 81.1,
|
|
"entropy": 0.40260612554848196,
|
|
"epoch": 0.2534685165421558,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 2.015625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0882080078125,
|
|
"num_tokens": 8849189.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.22220884561538695,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.22220885157585143,
|
|
"step": 1900,
|
|
"step_time": 16.079282514657827,
|
|
"zero_adv_skip_rate": 0.3895
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 490.5,
|
|
"completions/max_terminated_length": 292.1,
|
|
"completions/mean_length": 191.1875,
|
|
"completions/mean_terminated_length": 164.83214416503907,
|
|
"completions/min_length": 93.5,
|
|
"completions/min_terminated_length": 93.5,
|
|
"entropy": 0.3746134324464947,
|
|
"epoch": 0.2548025613660619,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.40234375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.002979572117328644,
|
|
"num_tokens": 8891356.0,
|
|
"reward": 0.4875,
|
|
"reward_std": 0.34637176990509033,
|
|
"rewards/correctness_reward/mean": 0.4875,
|
|
"rewards/correctness_reward/std": 0.3463717758655548,
|
|
"step": 1910,
|
|
"step_time": 13.087065459508448,
|
|
"zero_adv_skip_rate": 0.3885
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 332.5,
|
|
"completions/max_terminated_length": 332.5,
|
|
"completions/mean_length": 149.0,
|
|
"completions/mean_terminated_length": 149.0,
|
|
"completions/min_length": 66.5,
|
|
"completions/min_terminated_length": 66.5,
|
|
"entropy": 0.4830262143164873,
|
|
"epoch": 0.256136606189968,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.421875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.013924413919448852,
|
|
"num_tokens": 8934820.0,
|
|
"reward": 0.475,
|
|
"reward_std": 0.21674493551254273,
|
|
"rewards/correctness_reward/mean": 0.475,
|
|
"rewards/correctness_reward/std": 0.21674493551254273,
|
|
"step": 1920,
|
|
"step_time": 8.980367689486593,
|
|
"zero_adv_skip_rate": 0.3891
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 771.4,
|
|
"completions/max_terminated_length": 556.0,
|
|
"completions/mean_length": 298.1625,
|
|
"completions/mean_terminated_length": 256.45357818603514,
|
|
"completions/min_length": 95.6,
|
|
"completions/min_terminated_length": 95.6,
|
|
"entropy": 0.36281311362981794,
|
|
"epoch": 0.25747065101387406,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.004514070600271225,
|
|
"num_tokens": 8989561.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.19864802658557892,
|
|
"rewards/correctness_reward/mean": 0.1875,
|
|
"rewards/correctness_reward/std": 0.19864802658557892,
|
|
"step": 1930,
|
|
"step_time": 18.786238277237864,
|
|
"zero_adv_skip_rate": 0.3896
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 952.6,
|
|
"completions/max_terminated_length": 365.2,
|
|
"completions/mean_length": 237.75,
|
|
"completions/mean_terminated_length": 153.27143211364745,
|
|
"completions/min_length": 66.4,
|
|
"completions/min_terminated_length": 66.4,
|
|
"entropy": 0.2947356142103672,
|
|
"epoch": 0.25880469583778015,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.58984375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.034649628400802615,
|
|
"num_tokens": 9040181.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.23691109120845794,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.23691110908985138,
|
|
"step": 1940,
|
|
"step_time": 23.335378559306264,
|
|
"zero_adv_skip_rate": 0.3902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 705.4,
|
|
"completions/max_terminated_length": 530.5,
|
|
"completions/mean_length": 278.1375,
|
|
"completions/mean_terminated_length": 255.02678680419922,
|
|
"completions/min_length": 113.3,
|
|
"completions/min_terminated_length": 113.3,
|
|
"entropy": 0.4007451835786924,
|
|
"epoch": 0.26013874066168624,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.035489636659622195,
|
|
"num_tokens": 9095912.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.35013834834098817,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.3501383662223816,
|
|
"step": 1950,
|
|
"step_time": 17.852512687351556,
|
|
"zero_adv_skip_rate": 0.3892
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 541.5,
|
|
"completions/max_terminated_length": 385.3,
|
|
"completions/mean_length": 179.025,
|
|
"completions/mean_terminated_length": 156.4214294433594,
|
|
"completions/min_length": 64.1,
|
|
"completions/min_terminated_length": 64.1,
|
|
"entropy": 0.3958166448088984,
|
|
"epoch": 0.26147278548559233,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 1.2421875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.008100959658622741,
|
|
"num_tokens": 9137386.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.24662851691246032,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.2466285228729248,
|
|
"step": 1960,
|
|
"step_time": 13.928329987172038,
|
|
"zero_adv_skip_rate": 0.3893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 440.5,
|
|
"completions/max_terminated_length": 440.5,
|
|
"completions/mean_length": 195.4125,
|
|
"completions/mean_terminated_length": 195.4125,
|
|
"completions/min_length": 88.4,
|
|
"completions/min_terminated_length": 88.4,
|
|
"entropy": 0.33896021656692027,
|
|
"epoch": 0.2628068303094984,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.009104859828948975,
|
|
"num_tokens": 9180387.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.22051151394844054,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.2205115258693695,
|
|
"step": 1970,
|
|
"step_time": 11.504345609992743,
|
|
"zero_adv_skip_rate": 0.3898
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 719.5,
|
|
"completions/max_terminated_length": 650.2,
|
|
"completions/mean_length": 302.8375,
|
|
"completions/mean_terminated_length": 262.6833381652832,
|
|
"completions/min_length": 97.6,
|
|
"completions/min_terminated_length": 97.6,
|
|
"entropy": 0.3239540479844436,
|
|
"epoch": 0.2641408751334045,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.361328125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.00037808269262313844,
|
|
"num_tokens": 9234182.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.33204143941402436,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.3320414572954178,
|
|
"step": 1980,
|
|
"step_time": 18.24194540893659,
|
|
"zero_adv_skip_rate": 0.3889
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 765.9,
|
|
"completions/max_terminated_length": 486.6,
|
|
"completions/mean_length": 302.475,
|
|
"completions/mean_terminated_length": 260.8017913818359,
|
|
"completions/min_length": 104.4,
|
|
"completions/min_terminated_length": 104.4,
|
|
"entropy": 0.38056028336286546,
|
|
"epoch": 0.26547491995731054,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.69140625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.037492001056671144,
|
|
"num_tokens": 9288452.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.20580926835536956,
|
|
"rewards/correctness_reward/mean": 0.1875,
|
|
"rewards/correctness_reward/std": 0.20580926835536956,
|
|
"step": 1990,
|
|
"step_time": 18.341759141627698,
|
|
"zero_adv_skip_rate": 0.3894
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 545.4,
|
|
"completions/max_terminated_length": 402.9,
|
|
"completions/mean_length": 203.6875,
|
|
"completions/mean_terminated_length": 175.41607437133788,
|
|
"completions/min_length": 65.2,
|
|
"completions/min_terminated_length": 65.2,
|
|
"entropy": 0.3810936876107007,
|
|
"epoch": 0.26680896478121663,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 1.109375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.015564115345478058,
|
|
"num_tokens": 9331659.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.37032019793987275,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.3703202098608017,
|
|
"step": 2000,
|
|
"step_time": 14.299169551301748,
|
|
"zero_adv_skip_rate": 0.3885
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 558.7,
|
|
"completions/max_terminated_length": 314.3,
|
|
"completions/mean_length": 215.675,
|
|
"completions/mean_terminated_length": 182.31786346435547,
|
|
"completions/min_length": 94.1,
|
|
"completions/min_terminated_length": 94.1,
|
|
"entropy": 0.34085980989038944,
|
|
"epoch": 0.2681430096051227,
|
|
"frac_reward_zero_std": 0.9,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.018169008195400238,
|
|
"num_tokens": 9380969.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.05345224738121033,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.05345224738121033,
|
|
"step": 2010,
|
|
"step_time": 13.983635657373815,
|
|
"zero_adv_skip_rate": 0.391
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 1052.6,
|
|
"completions/max_terminated_length": 740.2,
|
|
"completions/mean_length": 376.5875,
|
|
"completions/mean_terminated_length": 333.0625,
|
|
"completions/min_length": 118.7,
|
|
"completions/min_terminated_length": 118.7,
|
|
"entropy": 0.28785491987530676,
|
|
"epoch": 0.2694770544290288,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.80078125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.187616765499115,
|
|
"num_tokens": 9438840.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.26388694643974303,
|
|
"rewards/correctness_reward/mean": 0.2125,
|
|
"rewards/correctness_reward/std": 0.26388695240020754,
|
|
"step": 2020,
|
|
"step_time": 25.619071828108282,
|
|
"zero_adv_skip_rate": 0.3906
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 427.6,
|
|
"completions/max_terminated_length": 422.9,
|
|
"completions/mean_length": 201.9375,
|
|
"completions/mean_terminated_length": 200.05535736083985,
|
|
"completions/min_length": 93.7,
|
|
"completions/min_terminated_length": 93.7,
|
|
"entropy": 0.33810059167444706,
|
|
"epoch": 0.2708110992529349,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.51171875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.013264515995979309,
|
|
"num_tokens": 9483963.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.22051151394844054,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.2205115258693695,
|
|
"step": 2030,
|
|
"step_time": 11.026826943736523,
|
|
"zero_adv_skip_rate": 0.3911
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 197.0,
|
|
"completions/max_terminated_length": 193.6,
|
|
"completions/mean_length": 120.0375,
|
|
"completions/mean_terminated_length": 118.98035774230956,
|
|
"completions/min_length": 59.3,
|
|
"completions/min_terminated_length": 59.3,
|
|
"entropy": 0.50812628865242,
|
|
"epoch": 0.272145144076841,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 1.078125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.03968772292137146,
|
|
"num_tokens": 9521822.0,
|
|
"reward": 0.475,
|
|
"reward_std": 0.2832020968198776,
|
|
"rewards/correctness_reward/mean": 0.475,
|
|
"rewards/correctness_reward/std": 0.28320211470127105,
|
|
"step": 2040,
|
|
"step_time": 5.9838259254582225,
|
|
"zero_adv_skip_rate": 0.3912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 791.0,
|
|
"completions/max_terminated_length": 678.5,
|
|
"completions/mean_length": 346.9,
|
|
"completions/mean_terminated_length": 325.33392944335935,
|
|
"completions/min_length": 89.9,
|
|
"completions/min_terminated_length": 89.9,
|
|
"entropy": 0.3214887392241508,
|
|
"epoch": 0.27347918890074707,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.017318445444107055,
|
|
"num_tokens": 9589758.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.18515617549419403,
|
|
"rewards/correctness_reward/mean": 0.2125,
|
|
"rewards/correctness_reward/std": 0.185156187415123,
|
|
"step": 2050,
|
|
"step_time": 19.30315113281831,
|
|
"zero_adv_skip_rate": 0.3922
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 714.4,
|
|
"completions/max_terminated_length": 413.6,
|
|
"completions/mean_length": 240.5,
|
|
"completions/mean_terminated_length": 196.43750457763673,
|
|
"completions/min_length": 79.6,
|
|
"completions/min_terminated_length": 79.6,
|
|
"entropy": 0.42352253509064514,
|
|
"epoch": 0.27481323372465316,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.6015625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0074240297079086305,
|
|
"num_tokens": 9639830.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.2777381867170334,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.27773819863796234,
|
|
"step": 2060,
|
|
"step_time": 17.75064854118973,
|
|
"zero_adv_skip_rate": 0.3922
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 789.2,
|
|
"completions/max_terminated_length": 422.6,
|
|
"completions/mean_length": 228.2375,
|
|
"completions/mean_terminated_length": 180.43214492797853,
|
|
"completions/min_length": 89.4,
|
|
"completions/min_terminated_length": 89.4,
|
|
"entropy": 0.45181905416150886,
|
|
"epoch": 0.27614727854855925,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.005916078388690948,
|
|
"num_tokens": 9686673.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.14056250154972078,
|
|
"rewards/correctness_reward/mean": 0.2,
|
|
"rewards/correctness_reward/std": 0.14056250751018523,
|
|
"step": 2070,
|
|
"step_time": 18.854873564839362,
|
|
"zero_adv_skip_rate": 0.3937
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 652.2,
|
|
"completions/max_terminated_length": 247.0,
|
|
"completions/mean_length": 191.4375,
|
|
"completions/mean_terminated_length": 138.1875015258789,
|
|
"completions/min_length": 75.4,
|
|
"completions/min_terminated_length": 75.4,
|
|
"entropy": 0.4336401326581836,
|
|
"epoch": 0.27748132337246534,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 1.0546875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04423301219940186,
|
|
"num_tokens": 9729844.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.24954409003257752,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.24954410195350646,
|
|
"step": 2080,
|
|
"step_time": 16.145107764471323,
|
|
"zero_adv_skip_rate": 0.3942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 750.9,
|
|
"completions/max_terminated_length": 586.0,
|
|
"completions/mean_length": 218.0625,
|
|
"completions/mean_terminated_length": 195.15714416503906,
|
|
"completions/min_length": 66.7,
|
|
"completions/min_terminated_length": 66.7,
|
|
"entropy": 0.4185833679512143,
|
|
"epoch": 0.27881536819637137,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.062458080053329465,
|
|
"num_tokens": 9776081.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.3817271083593369,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.38172711431980133,
|
|
"step": 2090,
|
|
"step_time": 19.10197654766962,
|
|
"zero_adv_skip_rate": 0.3928
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 583.3,
|
|
"completions/max_terminated_length": 496.8,
|
|
"completions/mean_length": 225.0375,
|
|
"completions/mean_terminated_length": 204.72678833007814,
|
|
"completions/min_length": 73.0,
|
|
"completions/min_terminated_length": 73.0,
|
|
"entropy": 0.4707152834162116,
|
|
"epoch": 0.28014941302027746,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6171875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.035109403729438785,
|
|
"num_tokens": 9822676.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.20411193668842315,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.20411194264888763,
|
|
"step": 2100,
|
|
"step_time": 14.536969464179128,
|
|
"zero_adv_skip_rate": 0.3933
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 710.2,
|
|
"completions/max_terminated_length": 327.9,
|
|
"completions/mean_length": 209.825,
|
|
"completions/mean_terminated_length": 159.84107360839843,
|
|
"completions/min_length": 84.1,
|
|
"completions/min_terminated_length": 84.1,
|
|
"entropy": 0.3511333417147398,
|
|
"epoch": 0.28148345784418355,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.16383631229400636,
|
|
"num_tokens": 9867798.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.23314451277256013,
|
|
"rewards/correctness_reward/mean": 0.45,
|
|
"rewards/correctness_reward/std": 0.2331445187330246,
|
|
"step": 2110,
|
|
"step_time": 17.468502074573188,
|
|
"zero_adv_skip_rate": 0.3938
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 380.8,
|
|
"completions/max_terminated_length": 350.3,
|
|
"completions/mean_length": 166.7875,
|
|
"completions/mean_terminated_length": 161.96428680419922,
|
|
"completions/min_length": 88.9,
|
|
"completions/min_terminated_length": 88.9,
|
|
"entropy": 0.40456744205827516,
|
|
"epoch": 0.28281750266808964,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.13901004791259766,
|
|
"num_tokens": 9909213.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.2575641840696335,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.25756419003009795,
|
|
"step": 2120,
|
|
"step_time": 10.138931676466019,
|
|
"zero_adv_skip_rate": 0.3939
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 542.2,
|
|
"completions/max_terminated_length": 526.1,
|
|
"completions/mean_length": 257.6375,
|
|
"completions/mean_terminated_length": 254.88214416503905,
|
|
"completions/min_length": 127.3,
|
|
"completions/min_terminated_length": 127.3,
|
|
"entropy": 0.405787813787659,
|
|
"epoch": 0.2841515474919957,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.46875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.008960254490375519,
|
|
"num_tokens": 9963840.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.13340125977993011,
|
|
"rewards/correctness_reward/mean": 0.125,
|
|
"rewards/correctness_reward/std": 0.1334012657403946,
|
|
"step": 2130,
|
|
"step_time": 13.39019814990461,
|
|
"zero_adv_skip_rate": 0.3953
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 449.6,
|
|
"completions/max_terminated_length": 427.7,
|
|
"completions/mean_length": 203.9,
|
|
"completions/mean_terminated_length": 199.32857284545898,
|
|
"completions/min_length": 74.1,
|
|
"completions/min_terminated_length": 74.1,
|
|
"entropy": 0.3403379522264004,
|
|
"epoch": 0.2854855923159018,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.10177725553512573,
|
|
"num_tokens": 10009888.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.21674493551254273,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.21674493551254273,
|
|
"step": 2140,
|
|
"step_time": 11.51300333328545,
|
|
"zero_adv_skip_rate": 0.3958
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 367.6,
|
|
"completions/max_terminated_length": 366.6,
|
|
"completions/mean_length": 167.4,
|
|
"completions/mean_terminated_length": 166.93035774230958,
|
|
"completions/min_length": 69.1,
|
|
"completions/min_terminated_length": 69.1,
|
|
"entropy": 0.40913843860228855,
|
|
"epoch": 0.2868196371398079,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.02297641634941101,
|
|
"num_tokens": 10059952.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.2739637613296509,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.27396377325057986,
|
|
"step": 2150,
|
|
"step_time": 9.857016847841441,
|
|
"zero_adv_skip_rate": 0.3958
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 595.4,
|
|
"completions/max_terminated_length": 594.9,
|
|
"completions/mean_length": 243.8125,
|
|
"completions/mean_terminated_length": 243.55357208251954,
|
|
"completions/min_length": 96.3,
|
|
"completions/min_terminated_length": 96.3,
|
|
"entropy": 0.3530616657808423,
|
|
"epoch": 0.288153681963714,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.39453125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.00621318519115448,
|
|
"num_tokens": 10110513.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.18515617549419403,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.185156187415123,
|
|
"step": 2160,
|
|
"step_time": 14.732966761291028,
|
|
"zero_adv_skip_rate": 0.3968
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 484.4,
|
|
"completions/max_terminated_length": 316.9,
|
|
"completions/mean_length": 208.7375,
|
|
"completions/mean_terminated_length": 183.70000381469725,
|
|
"completions/min_length": 90.6,
|
|
"completions/min_terminated_length": 90.6,
|
|
"entropy": 0.39839421389624474,
|
|
"epoch": 0.2894877267876201,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.2080078125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.005377960950136184,
|
|
"num_tokens": 10158100.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.4082238733768463,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.40822388529777526,
|
|
"step": 2170,
|
|
"step_time": 13.068823701422662,
|
|
"zero_adv_skip_rate": 0.3949
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 820.8,
|
|
"completions/max_terminated_length": 627.3,
|
|
"completions/mean_length": 367.4,
|
|
"completions/mean_terminated_length": 328.31964721679685,
|
|
"completions/min_length": 148.7,
|
|
"completions/min_terminated_length": 148.7,
|
|
"entropy": 0.47854255717247723,
|
|
"epoch": 0.29082177161152617,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.046568119525909425,
|
|
"num_tokens": 10216012.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.22051151394844054,
|
|
"rewards/correctness_reward/mean": 0.15,
|
|
"rewards/correctness_reward/std": 0.2205115258693695,
|
|
"step": 2180,
|
|
"step_time": 19.9710184074007,
|
|
"zero_adv_skip_rate": 0.3954
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 753.4,
|
|
"completions/max_terminated_length": 458.5,
|
|
"completions/mean_length": 258.6875,
|
|
"completions/mean_terminated_length": 216.37143249511718,
|
|
"completions/min_length": 85.5,
|
|
"completions/min_terminated_length": 85.5,
|
|
"entropy": 0.3784556833228895,
|
|
"epoch": 0.2921558164354322,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.017186205089092254,
|
|
"num_tokens": 10267651.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.29122219085693357,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.29122220277786254,
|
|
"step": 2190,
|
|
"step_time": 18.658431936427952,
|
|
"zero_adv_skip_rate": 0.395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 368.6,
|
|
"completions/max_terminated_length": 368.6,
|
|
"completions/mean_length": 195.7,
|
|
"completions/mean_terminated_length": 195.7,
|
|
"completions/min_length": 89.4,
|
|
"completions/min_terminated_length": 89.4,
|
|
"entropy": 0.41998405009508133,
|
|
"epoch": 0.2934898612593383,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.34765625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.03831990659236908,
|
|
"num_tokens": 10311771.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.2575641840696335,
|
|
"rewards/correctness_reward/mean": 0.175,
|
|
"rewards/correctness_reward/std": 0.25756419003009795,
|
|
"step": 2200,
|
|
"step_time": 9.842718903161586,
|
|
"zero_adv_skip_rate": 0.395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 771.5,
|
|
"completions/max_terminated_length": 591.9,
|
|
"completions/mean_length": 251.1625,
|
|
"completions/mean_terminated_length": 226.1107162475586,
|
|
"completions/min_length": 101.2,
|
|
"completions/min_terminated_length": 101.2,
|
|
"entropy": 0.40196328131215914,
|
|
"epoch": 0.2948239060832444,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.09417752027511597,
|
|
"num_tokens": 10359016.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.3274160087108612,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.32741602063179015,
|
|
"step": 2210,
|
|
"step_time": 19.153742548730225,
|
|
"zero_adv_skip_rate": 0.3946
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 620.8,
|
|
"completions/max_terminated_length": 286.9,
|
|
"completions/mean_length": 179.5,
|
|
"completions/mean_terminated_length": 130.67143135070802,
|
|
"completions/min_length": 69.8,
|
|
"completions/min_terminated_length": 69.8,
|
|
"entropy": 0.3901735465042293,
|
|
"epoch": 0.29615795090715047,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.10851356983184815,
|
|
"num_tokens": 10400832.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.32111361920833587,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.3211136251688004,
|
|
"step": 2220,
|
|
"step_time": 15.841997099854051,
|
|
"zero_adv_skip_rate": 0.3937
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 732.4,
|
|
"completions/max_terminated_length": 505.1,
|
|
"completions/mean_length": 270.9375,
|
|
"completions/mean_terminated_length": 228.02678833007812,
|
|
"completions/min_length": 97.8,
|
|
"completions/min_terminated_length": 97.8,
|
|
"entropy": 0.38630076094220084,
|
|
"epoch": 0.29749199573105656,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.3671875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0075136393308639525,
|
|
"num_tokens": 10455363.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.29752458035945895,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.29752459824085237,
|
|
"step": 2230,
|
|
"step_time": 18.106208264268936,
|
|
"zero_adv_skip_rate": 0.3937
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 483.3,
|
|
"completions/max_terminated_length": 352.0,
|
|
"completions/mean_length": 186.3375,
|
|
"completions/mean_terminated_length": 164.66250305175782,
|
|
"completions/min_length": 63.6,
|
|
"completions/min_terminated_length": 63.6,
|
|
"entropy": 0.41163213457912207,
|
|
"epoch": 0.29882604055496265,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.005924530327320099,
|
|
"num_tokens": 10497678.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.18515617549419403,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.185156187415123,
|
|
"step": 2240,
|
|
"step_time": 12.123462114855647,
|
|
"zero_adv_skip_rate": 0.3946
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 649.4,
|
|
"completions/max_terminated_length": 307.8,
|
|
"completions/mean_length": 188.725,
|
|
"completions/mean_terminated_length": 138.92500267028808,
|
|
"completions/min_length": 64.1,
|
|
"completions/min_terminated_length": 64.1,
|
|
"entropy": 0.41237155680677723,
|
|
"epoch": 0.30016008537886874,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.828125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.12128579616546631,
|
|
"num_tokens": 10542576.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.29839127957820893,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.29839127957820893,
|
|
"step": 2250,
|
|
"step_time": 16.354142129793765,
|
|
"zero_adv_skip_rate": 0.3942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 623.6,
|
|
"completions/max_terminated_length": 556.2,
|
|
"completions/mean_length": 232.9375,
|
|
"completions/mean_terminated_length": 212.08928985595702,
|
|
"completions/min_length": 87.3,
|
|
"completions/min_terminated_length": 87.3,
|
|
"entropy": 0.4015454820667704,
|
|
"epoch": 0.3014941302027748,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.33984375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.01879410743713379,
|
|
"num_tokens": 10589323.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.2558668524026871,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.25586686432361605,
|
|
"step": 2260,
|
|
"step_time": 16.13465596269816,
|
|
"zero_adv_skip_rate": 0.3942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 401.6,
|
|
"completions/max_terminated_length": 401.1,
|
|
"completions/mean_length": 175.85,
|
|
"completions/mean_terminated_length": 175.6839286804199,
|
|
"completions/min_length": 84.8,
|
|
"completions/min_terminated_length": 84.8,
|
|
"entropy": 0.45571103346134934,
|
|
"epoch": 0.3028281750266809,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.06149976253509522,
|
|
"num_tokens": 10632999.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.29291952252388,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.2929195284843445,
|
|
"step": 2270,
|
|
"step_time": 10.804162485525012,
|
|
"zero_adv_skip_rate": 0.3938
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 542.1,
|
|
"completions/max_terminated_length": 390.7,
|
|
"completions/mean_length": 232.3125,
|
|
"completions/mean_terminated_length": 210.92678833007812,
|
|
"completions/min_length": 112.7,
|
|
"completions/min_terminated_length": 112.7,
|
|
"entropy": 0.4133468729754289,
|
|
"epoch": 0.304162219850587,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.041517728567123414,
|
|
"num_tokens": 10680864.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.2756610929965973,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.27566109895706176,
|
|
"step": 2280,
|
|
"step_time": 14.203748985473066,
|
|
"zero_adv_skip_rate": 0.3939
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 761.6,
|
|
"completions/max_terminated_length": 402.9,
|
|
"completions/mean_length": 238.4625,
|
|
"completions/mean_terminated_length": 191.17797775268554,
|
|
"completions/min_length": 68.9,
|
|
"completions/min_terminated_length": 68.9,
|
|
"entropy": 0.32928502676077187,
|
|
"epoch": 0.3054962646744931,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 1.1484375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.25518844127655027,
|
|
"num_tokens": 10728413.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.3265775293111801,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.3265775412321091,
|
|
"step": 2290,
|
|
"step_time": 19.794006299506872,
|
|
"zero_adv_skip_rate": 0.393
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 599.3,
|
|
"completions/max_terminated_length": 440.8,
|
|
"completions/mean_length": 214.6375,
|
|
"completions/mean_terminated_length": 173.2017910003662,
|
|
"completions/min_length": 54.7,
|
|
"completions/min_terminated_length": 54.7,
|
|
"entropy": 0.3374497362722953,
|
|
"epoch": 0.3068303094983991,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04680472016334534,
|
|
"num_tokens": 10775928.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.27942767143249514,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.27942768931388856,
|
|
"step": 2300,
|
|
"step_time": 15.585032815299929,
|
|
"zero_adv_skip_rate": 0.393
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 612.4,
|
|
"completions/max_terminated_length": 280.0,
|
|
"completions/mean_length": 245.0625,
|
|
"completions/mean_terminated_length": 175.56667098999023,
|
|
"completions/min_length": 88.8,
|
|
"completions/min_terminated_length": 88.8,
|
|
"entropy": 0.39596075662929153,
|
|
"epoch": 0.3081643543223052,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.55078125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.018163371086120605,
|
|
"num_tokens": 10824357.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.38550153374671936,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.3855015397071838,
|
|
"step": 2310,
|
|
"step_time": 15.908127136155963,
|
|
"zero_adv_skip_rate": 0.3918
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 1033.2,
|
|
"completions/max_terminated_length": 781.4,
|
|
"completions/mean_length": 347.0625,
|
|
"completions/mean_terminated_length": 306.31250228881834,
|
|
"completions/min_length": 98.5,
|
|
"completions/min_terminated_length": 98.5,
|
|
"entropy": 0.39508737772703173,
|
|
"epoch": 0.3094983991462113,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.11924918889999389,
|
|
"num_tokens": 10883722.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.23861626982688905,
|
|
"rewards/correctness_reward/mean": 0.2,
|
|
"rewards/correctness_reward/std": 0.23861626982688905,
|
|
"step": 2320,
|
|
"step_time": 25.414644118025898,
|
|
"zero_adv_skip_rate": 0.3922
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 733.8,
|
|
"completions/max_terminated_length": 376.5,
|
|
"completions/mean_length": 219.525,
|
|
"completions/mean_terminated_length": 171.2250015258789,
|
|
"completions/min_length": 67.2,
|
|
"completions/min_terminated_length": 67.2,
|
|
"entropy": 0.4223176259547472,
|
|
"epoch": 0.3108324439701174,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.65625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04183002412319183,
|
|
"num_tokens": 10929996.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.29037118554115293,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.29037119150161744,
|
|
"step": 2330,
|
|
"step_time": 18.48113480526954,
|
|
"zero_adv_skip_rate": 0.3923
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 718.7,
|
|
"completions/max_terminated_length": 378.9,
|
|
"completions/mean_length": 219.3125,
|
|
"completions/mean_terminated_length": 172.05178756713866,
|
|
"completions/min_length": 80.5,
|
|
"completions/min_terminated_length": 80.5,
|
|
"entropy": 0.3718496221782906,
|
|
"epoch": 0.3121664887940235,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.1982421875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.07160024642944336,
|
|
"num_tokens": 10975581.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.2912300378084183,
|
|
"rewards/correctness_reward/mean": 0.3375,
|
|
"rewards/correctness_reward/std": 0.2912300378084183,
|
|
"step": 2340,
|
|
"step_time": 18.560864007472993,
|
|
"zero_adv_skip_rate": 0.3919
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 517.0,
|
|
"completions/max_terminated_length": 506.0,
|
|
"completions/mean_length": 176.5,
|
|
"completions/mean_terminated_length": 175.02142868041992,
|
|
"completions/min_length": 67.4,
|
|
"completions/min_terminated_length": 67.4,
|
|
"entropy": 0.33584841593567816,
|
|
"epoch": 0.31350053361792957,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.13407086133956908,
|
|
"num_tokens": 11026765.0,
|
|
"reward": 0.5625,
|
|
"reward_std": 0.2802865236997604,
|
|
"rewards/correctness_reward/mean": 0.5625,
|
|
"rewards/correctness_reward/std": 0.2802865356206894,
|
|
"step": 2350,
|
|
"step_time": 13.667250429466367,
|
|
"zero_adv_skip_rate": 0.3915
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 551.9,
|
|
"completions/max_terminated_length": 525.0,
|
|
"completions/mean_length": 199.95,
|
|
"completions/mean_terminated_length": 191.94464492797852,
|
|
"completions/min_length": 73.1,
|
|
"completions/min_terminated_length": 73.1,
|
|
"entropy": 0.32566666905768216,
|
|
"epoch": 0.31483457844183566,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.4765625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1297295570373535,
|
|
"num_tokens": 11070505.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.3774814397096634,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.3774814516305923,
|
|
"step": 2360,
|
|
"step_time": 14.743884199298918,
|
|
"zero_adv_skip_rate": 0.3907
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 491.4,
|
|
"completions/max_terminated_length": 305.1,
|
|
"completions/mean_length": 215.15,
|
|
"completions/mean_terminated_length": 190.8732147216797,
|
|
"completions/min_length": 85.3,
|
|
"completions/min_terminated_length": 85.3,
|
|
"entropy": 0.3841290380805731,
|
|
"epoch": 0.31616862326574174,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04240175187587738,
|
|
"num_tokens": 11115565.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.22051151394844054,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.2205115258693695,
|
|
"step": 2370,
|
|
"step_time": 12.693473045434803,
|
|
"zero_adv_skip_rate": 0.3911
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 630.3,
|
|
"completions/max_terminated_length": 439.3,
|
|
"completions/mean_length": 222.4625,
|
|
"completions/mean_terminated_length": 195.53750305175782,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 0.3607313189422712,
|
|
"epoch": 0.31750266808964783,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.90625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.07302526235580445,
|
|
"num_tokens": 11162082.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.3282748609781265,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.328274866938591,
|
|
"step": 2380,
|
|
"step_time": 16.37717768298462,
|
|
"zero_adv_skip_rate": 0.3903
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 734.9,
|
|
"completions/max_terminated_length": 423.1,
|
|
"completions/mean_length": 232.7375,
|
|
"completions/mean_terminated_length": 187.37143096923828,
|
|
"completions/min_length": 77.6,
|
|
"completions/min_terminated_length": 77.6,
|
|
"entropy": 0.3480307888239622,
|
|
"epoch": 0.3188367129135539,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.002599151059985161,
|
|
"num_tokens": 11216197.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.14056250154972078,
|
|
"rewards/correctness_reward/mean": 0.225,
|
|
"rewards/correctness_reward/std": 0.14056250751018523,
|
|
"step": 2390,
|
|
"step_time": 18.411284341104327,
|
|
"zero_adv_skip_rate": 0.3916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 525.5,
|
|
"completions/max_terminated_length": 525.0,
|
|
"completions/mean_length": 213.55,
|
|
"completions/mean_terminated_length": 212.75,
|
|
"completions/min_length": 76.6,
|
|
"completions/min_terminated_length": 76.6,
|
|
"entropy": 0.33944725152105093,
|
|
"epoch": 0.32017075773745995,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.07320928573608398,
|
|
"num_tokens": 11261505.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.3093190997838974,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.30931911170482634,
|
|
"step": 2400,
|
|
"step_time": 14.080355227552355,
|
|
"zero_adv_skip_rate": 0.3912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 659.9,
|
|
"completions/max_terminated_length": 327.1,
|
|
"completions/mean_length": 231.85,
|
|
"completions/mean_terminated_length": 184.81964492797852,
|
|
"completions/min_length": 85.8,
|
|
"completions/min_terminated_length": 85.8,
|
|
"entropy": 0.32968801592609714,
|
|
"epoch": 0.32150480256136604,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.51953125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.08805406093597412,
|
|
"num_tokens": 11310685.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.2946168541908264,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.2946168541908264,
|
|
"step": 2410,
|
|
"step_time": 18.30642430316657,
|
|
"zero_adv_skip_rate": 0.3909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 267.0,
|
|
"completions/max_terminated_length": 266.2,
|
|
"completions/mean_length": 102.8,
|
|
"completions/mean_terminated_length": 102.44285736083984,
|
|
"completions/min_length": 54.4,
|
|
"completions/min_terminated_length": 54.4,
|
|
"entropy": 0.32890814300626514,
|
|
"epoch": 0.32283884738527213,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.05751598477363586,
|
|
"num_tokens": 11347677.0,
|
|
"reward": 0.7875,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.7875,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 2420,
|
|
"step_time": 8.030000335164369,
|
|
"zero_adv_skip_rate": 0.3913
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 406.7,
|
|
"completions/max_terminated_length": 406.7,
|
|
"completions/mean_length": 197.875,
|
|
"completions/mean_terminated_length": 197.875,
|
|
"completions/min_length": 72.3,
|
|
"completions/min_terminated_length": 72.3,
|
|
"entropy": 0.37554805104931194,
|
|
"epoch": 0.3241728922091782,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.02026194930076599,
|
|
"num_tokens": 11391803.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.2739637613296509,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.27396377325057986,
|
|
"step": 2430,
|
|
"step_time": 11.395333940815181,
|
|
"zero_adv_skip_rate": 0.3914
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 619.0,
|
|
"completions/max_terminated_length": 318.6,
|
|
"completions/mean_length": 227.3625,
|
|
"completions/mean_terminated_length": 182.25714683532715,
|
|
"completions/min_length": 94.9,
|
|
"completions/min_terminated_length": 94.9,
|
|
"entropy": 0.3846766487695277,
|
|
"epoch": 0.3255069370330843,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.55859375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.06088706851005554,
|
|
"num_tokens": 11437376.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.18685350716114044,
|
|
"rewards/correctness_reward/mean": 0.45,
|
|
"rewards/correctness_reward/std": 0.18685351312160492,
|
|
"step": 2440,
|
|
"step_time": 16.4687256754376,
|
|
"zero_adv_skip_rate": 0.3922
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 310.3,
|
|
"completions/max_terminated_length": 308.4,
|
|
"completions/mean_length": 156.4375,
|
|
"completions/mean_terminated_length": 155.75714416503905,
|
|
"completions/min_length": 84.6,
|
|
"completions/min_terminated_length": 84.6,
|
|
"entropy": 0.3472076558641025,
|
|
"epoch": 0.3268409818569904,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.01564549207687378,
|
|
"num_tokens": 11476043.0,
|
|
"reward": 0.475,
|
|
"reward_std": 0.3202547669410706,
|
|
"rewards/correctness_reward/mean": 0.475,
|
|
"rewards/correctness_reward/std": 0.3202547788619995,
|
|
"step": 2450,
|
|
"step_time": 9.171368810441345,
|
|
"zero_adv_skip_rate": 0.3918
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 613.2,
|
|
"completions/max_terminated_length": 287.4,
|
|
"completions/mean_length": 190.425,
|
|
"completions/mean_terminated_length": 143.81250381469727,
|
|
"completions/min_length": 76.4,
|
|
"completions/min_terminated_length": 76.4,
|
|
"entropy": 0.40860809781588614,
|
|
"epoch": 0.3281750266808965,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1171875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.06592695713043213,
|
|
"num_tokens": 11519613.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.4082238733768463,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.40822388529777526,
|
|
"step": 2460,
|
|
"step_time": 16.867193171940745,
|
|
"zero_adv_skip_rate": 0.3902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 490.6,
|
|
"completions/max_terminated_length": 371.6,
|
|
"completions/mean_length": 193.45,
|
|
"completions/mean_terminated_length": 173.35714874267578,
|
|
"completions/min_length": 80.5,
|
|
"completions/min_terminated_length": 80.5,
|
|
"entropy": 0.39370156744761126,
|
|
"epoch": 0.3295090715048026,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.78125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.021271833777427675,
|
|
"num_tokens": 11562657.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.2857582807540894,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.28575828671455383,
|
|
"step": 2470,
|
|
"step_time": 13.338893197197468,
|
|
"zero_adv_skip_rate": 0.3899
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 816.7,
|
|
"completions/max_terminated_length": 525.7,
|
|
"completions/mean_length": 285.55,
|
|
"completions/mean_terminated_length": 239.36607360839844,
|
|
"completions/min_length": 86.1,
|
|
"completions/min_terminated_length": 86.1,
|
|
"entropy": 0.4412103972087304,
|
|
"epoch": 0.33084311632870866,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.018189245462417604,
|
|
"num_tokens": 11614853.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.12246559262275696,
|
|
"rewards/correctness_reward/mean": 0.1875,
|
|
"rewards/correctness_reward/std": 0.12246559858322144,
|
|
"step": 2480,
|
|
"step_time": 20.549430758226663,
|
|
"zero_adv_skip_rate": 0.3911
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 640.7,
|
|
"completions/max_terminated_length": 453.7,
|
|
"completions/mean_length": 257.1125,
|
|
"completions/mean_terminated_length": 232.3982147216797,
|
|
"completions/min_length": 82.5,
|
|
"completions/min_terminated_length": 82.5,
|
|
"entropy": 0.3490378811955452,
|
|
"epoch": 0.33217716115261475,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.025202518701553343,
|
|
"num_tokens": 11663070.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.20958369374275207,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.20958369374275207,
|
|
"step": 2490,
|
|
"step_time": 16.550165260303764,
|
|
"zero_adv_skip_rate": 0.3916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 718.4,
|
|
"completions/max_terminated_length": 607.1,
|
|
"completions/mean_length": 313.375,
|
|
"completions/mean_terminated_length": 271.4309555053711,
|
|
"completions/min_length": 108.6,
|
|
"completions/min_terminated_length": 108.6,
|
|
"entropy": 0.3616929804640157,
|
|
"epoch": 0.3335112059765208,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.69140625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.09250377416610718,
|
|
"num_tokens": 11716460.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.336654344201088,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.3366543620824814,
|
|
"step": 2500,
|
|
"step_time": 18.620791855454446,
|
|
"zero_adv_skip_rate": 0.3912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 463.9,
|
|
"completions/max_terminated_length": 447.8,
|
|
"completions/mean_length": 164.775,
|
|
"completions/mean_terminated_length": 159.7696434020996,
|
|
"completions/min_length": 70.6,
|
|
"completions/min_terminated_length": 70.6,
|
|
"entropy": 0.315380484983325,
|
|
"epoch": 0.3348452508004269,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.35546875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04680852890014649,
|
|
"num_tokens": 11757714.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 2510,
|
|
"step_time": 12.35833112616092,
|
|
"zero_adv_skip_rate": 0.3916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 723.4,
|
|
"completions/max_terminated_length": 392.9,
|
|
"completions/mean_length": 250.1875,
|
|
"completions/mean_terminated_length": 196.60893020629882,
|
|
"completions/min_length": 86.0,
|
|
"completions/min_terminated_length": 86.0,
|
|
"entropy": 0.43987293483223766,
|
|
"epoch": 0.33617929562433296,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 1.2109375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.014478948712348939,
|
|
"num_tokens": 11804169.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.39010659158229827,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.3901066094636917,
|
|
"step": 2520,
|
|
"step_time": 18.702291745785622,
|
|
"zero_adv_skip_rate": 0.3909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 568.5,
|
|
"completions/max_terminated_length": 348.3,
|
|
"completions/mean_length": 198.6375,
|
|
"completions/mean_terminated_length": 167.9428581237793,
|
|
"completions/min_length": 64.7,
|
|
"completions/min_terminated_length": 64.7,
|
|
"entropy": 0.42075373604893684,
|
|
"epoch": 0.33751334044823905,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.037510272860527036,
|
|
"num_tokens": 11850356.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.23691109120845794,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.23691110908985138,
|
|
"step": 2530,
|
|
"step_time": 14.678406396787613,
|
|
"zero_adv_skip_rate": 0.3913
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 139.8,
|
|
"completions/max_terminated_length": 135.7,
|
|
"completions/mean_length": 89.4125,
|
|
"completions/mean_terminated_length": 87.65178718566895,
|
|
"completions/min_length": 55.0,
|
|
"completions/min_terminated_length": 55.0,
|
|
"entropy": 0.39927145605906844,
|
|
"epoch": 0.33884738527214514,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.8671875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.02905937135219574,
|
|
"num_tokens": 11887317.0,
|
|
"reward": 0.6,
|
|
"reward_std": 0.38634001314640043,
|
|
"rewards/correctness_reward/mean": 0.6,
|
|
"rewards/correctness_reward/std": 0.38634001910686494,
|
|
"step": 2540,
|
|
"step_time": 5.355950976256281,
|
|
"zero_adv_skip_rate": 0.3906
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 348.3,
|
|
"completions/max_terminated_length": 348.3,
|
|
"completions/mean_length": 166.825,
|
|
"completions/mean_terminated_length": 166.825,
|
|
"completions/min_length": 78.8,
|
|
"completions/min_terminated_length": 78.8,
|
|
"entropy": 0.4082541119839464,
|
|
"epoch": 0.34018143009605123,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.05285796523094177,
|
|
"num_tokens": 11928751.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.32402134537696836,
|
|
"rewards/correctness_reward/mean": 0.5,
|
|
"rewards/correctness_reward/std": 0.3240213692188263,
|
|
"step": 2550,
|
|
"step_time": 9.880698113143444,
|
|
"zero_adv_skip_rate": 0.3902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 630.2,
|
|
"completions/max_terminated_length": 500.2,
|
|
"completions/mean_length": 304.125,
|
|
"completions/mean_terminated_length": 251.1684555053711,
|
|
"completions/min_length": 84.4,
|
|
"completions/min_terminated_length": 84.4,
|
|
"entropy": 0.2728805072672133,
|
|
"epoch": 0.3415154749199573,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.93359375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.03230591416358948,
|
|
"num_tokens": 11981233.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.3021578580141068,
|
|
"rewards/correctness_reward/mean": 0.3375,
|
|
"rewards/correctness_reward/std": 0.3021578699350357,
|
|
"step": 2560,
|
|
"step_time": 16.56249888949096,
|
|
"zero_adv_skip_rate": 0.3898
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 585.1,
|
|
"completions/max_terminated_length": 410.4,
|
|
"completions/mean_length": 214.325,
|
|
"completions/mean_terminated_length": 188.58035888671876,
|
|
"completions/min_length": 88.4,
|
|
"completions/min_terminated_length": 88.4,
|
|
"entropy": 0.28902783361263573,
|
|
"epoch": 0.3428495197438634,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.6328125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.03605950772762299,
|
|
"num_tokens": 12026843.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.15782093107700348,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.15782093703746797,
|
|
"step": 2570,
|
|
"step_time": 15.02198232281953,
|
|
"zero_adv_skip_rate": 0.3907
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 469.7,
|
|
"completions/max_terminated_length": 278.9,
|
|
"completions/mean_length": 171.5,
|
|
"completions/mean_terminated_length": 144.34643096923827,
|
|
"completions/min_length": 76.4,
|
|
"completions/min_terminated_length": 76.4,
|
|
"entropy": 0.32011016613493365,
|
|
"epoch": 0.3441835645677695,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.84375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.009830579161643982,
|
|
"num_tokens": 12068315.0,
|
|
"reward": 0.5875,
|
|
"reward_std": 0.3564689576625824,
|
|
"rewards/correctness_reward/mean": 0.5875,
|
|
"rewards/correctness_reward/std": 0.35646896362304686,
|
|
"step": 2580,
|
|
"step_time": 13.080226065125316,
|
|
"zero_adv_skip_rate": 0.3895
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 466.5,
|
|
"completions/max_terminated_length": 466.0,
|
|
"completions/mean_length": 206.9625,
|
|
"completions/mean_terminated_length": 206.67142868041992,
|
|
"completions/min_length": 83.2,
|
|
"completions/min_terminated_length": 83.2,
|
|
"entropy": 0.41039734343066814,
|
|
"epoch": 0.3455176093916756,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.75390625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.06756870746612549,
|
|
"num_tokens": 12118024.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.4082238733768463,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.40822388529777526,
|
|
"step": 2590,
|
|
"step_time": 13.097948760911823,
|
|
"zero_adv_skip_rate": 0.388
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 409.5,
|
|
"completions/max_terminated_length": 406.2,
|
|
"completions/mean_length": 200.5875,
|
|
"completions/mean_terminated_length": 199.61607208251954,
|
|
"completions/min_length": 77.5,
|
|
"completions/min_terminated_length": 77.5,
|
|
"entropy": 0.34765408392995595,
|
|
"epoch": 0.3468516542155816,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.98046875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04933868050575256,
|
|
"num_tokens": 12160975.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 2600,
|
|
"step_time": 11.128793311305344,
|
|
"zero_adv_skip_rate": 0.3885
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 361.5,
|
|
"completions/max_terminated_length": 359.4,
|
|
"completions/mean_length": 169.5875,
|
|
"completions/mean_terminated_length": 169.04285736083983,
|
|
"completions/min_length": 79.9,
|
|
"completions/min_terminated_length": 79.9,
|
|
"entropy": 0.39941130578517914,
|
|
"epoch": 0.3481856990394877,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0371873140335083,
|
|
"num_tokens": 12203646.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.3190365254878998,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.3190365254878998,
|
|
"step": 2610,
|
|
"step_time": 10.372189127095044,
|
|
"zero_adv_skip_rate": 0.3877
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 593.4,
|
|
"completions/max_terminated_length": 221.1,
|
|
"completions/mean_length": 172.6,
|
|
"completions/mean_terminated_length": 119.95893096923828,
|
|
"completions/min_length": 57.1,
|
|
"completions/min_terminated_length": 57.1,
|
|
"entropy": 0.420238196849823,
|
|
"epoch": 0.3495197438633938,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 1.140625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.010741403698921204,
|
|
"num_tokens": 12247382.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.4128367781639099,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.41283679008483887,
|
|
"step": 2620,
|
|
"step_time": 16.43125662440434,
|
|
"zero_adv_skip_rate": 0.3866
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 429.1,
|
|
"completions/max_terminated_length": 258.8,
|
|
"completions/mean_length": 167.1375,
|
|
"completions/mean_terminated_length": 144.13928680419923,
|
|
"completions/min_length": 76.6,
|
|
"completions/min_terminated_length": 76.6,
|
|
"entropy": 0.42276985733769834,
|
|
"epoch": 0.3508537886872999,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.73046875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.05066453814506531,
|
|
"num_tokens": 12289425.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.31564186215400697,
|
|
"rewards/correctness_reward/mean": 0.5,
|
|
"rewards/correctness_reward/std": 0.31564187407493594,
|
|
"step": 2630,
|
|
"step_time": 12.072963674366473,
|
|
"zero_adv_skip_rate": 0.3859
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 466.8,
|
|
"completions/max_terminated_length": 466.8,
|
|
"completions/mean_length": 213.675,
|
|
"completions/mean_terminated_length": 213.675,
|
|
"completions/min_length": 92.5,
|
|
"completions/min_terminated_length": 92.5,
|
|
"entropy": 0.44597601890563965,
|
|
"epoch": 0.35218783351120597,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04166600704193115,
|
|
"num_tokens": 12335199.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.1776151716709137,
|
|
"rewards/correctness_reward/mean": 0.225,
|
|
"rewards/correctness_reward/std": 0.1776151716709137,
|
|
"step": 2640,
|
|
"step_time": 12.331505982670933,
|
|
"zero_adv_skip_rate": 0.3867
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 747.8,
|
|
"completions/max_terminated_length": 373.1,
|
|
"completions/mean_length": 221.3375,
|
|
"completions/mean_terminated_length": 173.49285888671875,
|
|
"completions/min_length": 85.2,
|
|
"completions/min_terminated_length": 85.2,
|
|
"entropy": 0.4933831023850611,
|
|
"epoch": 0.35352187833511206,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.412109375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.09165474772453308,
|
|
"num_tokens": 12384098.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.31479085683822633,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.3147908627986908,
|
|
"step": 2650,
|
|
"step_time": 19.43178566461429,
|
|
"zero_adv_skip_rate": 0.3864
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 764.3,
|
|
"completions/max_terminated_length": 385.2,
|
|
"completions/mean_length": 217.775,
|
|
"completions/mean_terminated_length": 168.30000228881835,
|
|
"completions/min_length": 77.9,
|
|
"completions/min_terminated_length": 77.9,
|
|
"entropy": 0.39707687008194625,
|
|
"epoch": 0.35485592315901815,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.5390625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.24883441925048827,
|
|
"num_tokens": 12432336.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.3610740154981613,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.36107403337955474,
|
|
"step": 2660,
|
|
"step_time": 20.07360511254519,
|
|
"zero_adv_skip_rate": 0.3857
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 726.1,
|
|
"completions/max_terminated_length": 463.5,
|
|
"completions/mean_length": 254.925,
|
|
"completions/mean_terminated_length": 209.96786041259764,
|
|
"completions/min_length": 89.0,
|
|
"completions/min_terminated_length": 89.0,
|
|
"entropy": 0.391228948906064,
|
|
"epoch": 0.35618996798292424,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.77734375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.036087924242019655,
|
|
"num_tokens": 12482682.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.22598327100276946,
|
|
"rewards/correctness_reward/mean": 0.2,
|
|
"rewards/correctness_reward/std": 0.22598327696323395,
|
|
"step": 2670,
|
|
"step_time": 18.7566194367595,
|
|
"zero_adv_skip_rate": 0.3861
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 828.3,
|
|
"completions/max_terminated_length": 703.9,
|
|
"completions/mean_length": 279.1,
|
|
"completions/mean_terminated_length": 239.88214645385742,
|
|
"completions/min_length": 77.9,
|
|
"completions/min_terminated_length": 77.9,
|
|
"entropy": 0.33814236470498144,
|
|
"epoch": 0.3575240128068303,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.734375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.02455439567565918,
|
|
"num_tokens": 12534138.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.2293700873851776,
|
|
"rewards/correctness_reward/mean": 0.4375,
|
|
"rewards/correctness_reward/std": 0.2293700933456421,
|
|
"step": 2680,
|
|
"step_time": 21.235377711057662,
|
|
"zero_adv_skip_rate": 0.3866
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 694.3,
|
|
"completions/max_terminated_length": 694.3,
|
|
"completions/mean_length": 235.1625,
|
|
"completions/mean_terminated_length": 235.1625,
|
|
"completions/min_length": 89.9,
|
|
"completions/min_terminated_length": 89.9,
|
|
"entropy": 0.31704418116714805,
|
|
"epoch": 0.3588580576307364,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.70703125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.028187111020088196,
|
|
"num_tokens": 12580247.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.3737070143222809,
|
|
"rewards/correctness_reward/mean": 0.45,
|
|
"rewards/correctness_reward/std": 0.37370702624320984,
|
|
"step": 2690,
|
|
"step_time": 18.309814362507314,
|
|
"zero_adv_skip_rate": 0.3859
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 664.6,
|
|
"completions/max_terminated_length": 495.8,
|
|
"completions/mean_length": 254.7125,
|
|
"completions/mean_terminated_length": 231.61607208251954,
|
|
"completions/min_length": 123.9,
|
|
"completions/min_terminated_length": 123.9,
|
|
"entropy": 0.3292283690534532,
|
|
"epoch": 0.36019210245464245,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.56640625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.063671875,
|
|
"num_tokens": 12627784.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.3446744382381439,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.3446744501590729,
|
|
"step": 2700,
|
|
"step_time": 17.78281285734847,
|
|
"zero_adv_skip_rate": 0.3852
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 277.3,
|
|
"completions/max_terminated_length": 264.1,
|
|
"completions/mean_length": 152.25,
|
|
"completions/mean_terminated_length": 149.76785850524902,
|
|
"completions/min_length": 85.2,
|
|
"completions/min_terminated_length": 85.2,
|
|
"entropy": 0.3134821254760027,
|
|
"epoch": 0.36152614727854854,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.640625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.03380615711212158,
|
|
"num_tokens": 12667676.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 2710,
|
|
"step_time": 7.9624558936804535,
|
|
"zero_adv_skip_rate": 0.3856
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 703.0,
|
|
"completions/max_terminated_length": 436.1,
|
|
"completions/mean_length": 261.1625,
|
|
"completions/mean_terminated_length": 217.48036155700683,
|
|
"completions/min_length": 82.3,
|
|
"completions/min_terminated_length": 82.3,
|
|
"entropy": 0.35548358634114263,
|
|
"epoch": 0.3628601921024546,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.06987702250480651,
|
|
"num_tokens": 12717689.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.22051936089992524,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.22051936089992524,
|
|
"step": 2720,
|
|
"step_time": 18.132624627929182,
|
|
"zero_adv_skip_rate": 0.386
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0875,
|
|
"completions/max_length": 1446.1,
|
|
"completions/max_terminated_length": 698.8,
|
|
"completions/mean_length": 415.15,
|
|
"completions/mean_terminated_length": 287.98036880493163,
|
|
"completions/min_length": 106.7,
|
|
"completions/min_terminated_length": 106.7,
|
|
"entropy": 0.3836316540837288,
|
|
"epoch": 0.3641942369263607,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.490234375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.20934603214263917,
|
|
"num_tokens": 12784285.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.30639100074768066,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.30639100074768066,
|
|
"step": 2730,
|
|
"step_time": 36.23340278565884,
|
|
"zero_adv_skip_rate": 0.3861
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 354.6,
|
|
"completions/max_terminated_length": 336.0,
|
|
"completions/mean_length": 180.1875,
|
|
"completions/mean_terminated_length": 176.22321548461915,
|
|
"completions/min_length": 74.4,
|
|
"completions/min_terminated_length": 74.4,
|
|
"entropy": 0.3628197363577783,
|
|
"epoch": 0.3655282817502668,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.412109375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.09152626395225524,
|
|
"num_tokens": 12827524.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.36653792560100557,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.36653794944286344,
|
|
"step": 2740,
|
|
"step_time": 10.644024395477027,
|
|
"zero_adv_skip_rate": 0.3854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 796.7,
|
|
"completions/max_terminated_length": 274.3,
|
|
"completions/mean_length": 236.0625,
|
|
"completions/mean_terminated_length": 162.59464416503906,
|
|
"completions/min_length": 89.2,
|
|
"completions/min_terminated_length": 89.2,
|
|
"entropy": 0.35633380258721964,
|
|
"epoch": 0.3668623265741729,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.08652791380882263,
|
|
"num_tokens": 12876409.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.3021578580141068,
|
|
"rewards/correctness_reward/mean": 0.4375,
|
|
"rewards/correctness_reward/std": 0.3021578699350357,
|
|
"step": 2750,
|
|
"step_time": 22.940446787141262,
|
|
"zero_adv_skip_rate": 0.3851
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 736.1,
|
|
"completions/max_terminated_length": 565.2,
|
|
"completions/mean_length": 283.975,
|
|
"completions/mean_terminated_length": 258.84286041259764,
|
|
"completions/min_length": 100.3,
|
|
"completions/min_terminated_length": 100.3,
|
|
"entropy": 0.3635556264780462,
|
|
"epoch": 0.368196371398079,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.470703125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.012823998928070068,
|
|
"num_tokens": 12927191.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.18138959705829621,
|
|
"rewards/correctness_reward/mean": 0.1625,
|
|
"rewards/correctness_reward/std": 0.18138959705829621,
|
|
"step": 2760,
|
|
"step_time": 20.352181917242707,
|
|
"zero_adv_skip_rate": 0.3859
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 549.0,
|
|
"completions/max_terminated_length": 351.8,
|
|
"completions/mean_length": 212.5375,
|
|
"completions/mean_terminated_length": 183.13035888671874,
|
|
"completions/min_length": 64.6,
|
|
"completions/min_terminated_length": 64.6,
|
|
"entropy": 0.3989359430658321,
|
|
"epoch": 0.36953041622198507,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.5859375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.03282874524593353,
|
|
"num_tokens": 12971442.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.25210027396678925,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.25210027396678925,
|
|
"step": 2770,
|
|
"step_time": 16.269827077444642,
|
|
"zero_adv_skip_rate": 0.3859
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 524.0,
|
|
"completions/max_terminated_length": 452.7,
|
|
"completions/mean_length": 239.4875,
|
|
"completions/mean_terminated_length": 219.22678833007814,
|
|
"completions/min_length": 94.2,
|
|
"completions/min_terminated_length": 94.2,
|
|
"entropy": 0.2922306186519563,
|
|
"epoch": 0.37086446104589116,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.65625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.00011521447449922562,
|
|
"num_tokens": 13018337.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.2886660069227219,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.28866603076457975,
|
|
"step": 2780,
|
|
"step_time": 14.282013952359558,
|
|
"zero_adv_skip_rate": 0.386
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 824.0,
|
|
"completions/max_terminated_length": 525.2,
|
|
"completions/mean_length": 319.275,
|
|
"completions/mean_terminated_length": 276.1821479797363,
|
|
"completions/min_length": 109.1,
|
|
"completions/min_terminated_length": 109.1,
|
|
"entropy": 0.3863722750296195,
|
|
"epoch": 0.37219850586979725,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.06737493872642517,
|
|
"num_tokens": 13074703.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.13509859144687653,
|
|
"rewards/correctness_reward/mean": 0.3375,
|
|
"rewards/correctness_reward/std": 0.13509859144687653,
|
|
"step": 2790,
|
|
"step_time": 21.264455716777594,
|
|
"zero_adv_skip_rate": 0.3871
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 766.4,
|
|
"completions/max_terminated_length": 445.9,
|
|
"completions/mean_length": 245.425,
|
|
"completions/mean_terminated_length": 197.34464492797852,
|
|
"completions/min_length": 84.1,
|
|
"completions/min_terminated_length": 84.1,
|
|
"entropy": 0.2896826670039445,
|
|
"epoch": 0.37353255069370334,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.07472701668739319,
|
|
"num_tokens": 13126729.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.1977891743183136,
|
|
"rewards/correctness_reward/mean": 0.2125,
|
|
"rewards/correctness_reward/std": 0.19778918027877807,
|
|
"step": 2800,
|
|
"step_time": 20.112130511924626,
|
|
"zero_adv_skip_rate": 0.3879
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 214.1,
|
|
"completions/max_terminated_length": 213.6,
|
|
"completions/mean_length": 128.4875,
|
|
"completions/mean_terminated_length": 127.88392944335938,
|
|
"completions/min_length": 70.2,
|
|
"completions/min_terminated_length": 70.2,
|
|
"entropy": 0.34240490158221554,
|
|
"epoch": 0.37486659551760937,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.58203125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.015664732456207274,
|
|
"num_tokens": 13164584.0,
|
|
"reward": 0.5125,
|
|
"reward_std": 0.30762961506843567,
|
|
"rewards/correctness_reward/mean": 0.5125,
|
|
"rewards/correctness_reward/std": 0.3076296210289001,
|
|
"step": 2810,
|
|
"step_time": 6.960458372347057,
|
|
"zero_adv_skip_rate": 0.3875
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 701.8,
|
|
"completions/max_terminated_length": 570.5,
|
|
"completions/mean_length": 244.5125,
|
|
"completions/mean_terminated_length": 223.22321701049805,
|
|
"completions/min_length": 86.7,
|
|
"completions/min_terminated_length": 86.7,
|
|
"entropy": 0.3874131937821706,
|
|
"epoch": 0.37620064034151546,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0014697927981615067,
|
|
"num_tokens": 13216129.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.12793734967708587,
|
|
"rewards/correctness_reward/mean": 0.2375,
|
|
"rewards/correctness_reward/std": 0.12793734967708587,
|
|
"step": 2820,
|
|
"step_time": 18.406382846925407,
|
|
"zero_adv_skip_rate": 0.3887
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 885.8,
|
|
"completions/max_terminated_length": 847.8,
|
|
"completions/mean_length": 339.1625,
|
|
"completions/mean_terminated_length": 321.8571472167969,
|
|
"completions/min_length": 98.4,
|
|
"completions/min_terminated_length": 98.4,
|
|
"entropy": 0.34475863608531654,
|
|
"epoch": 0.37753468516542155,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.328125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0018888048827648163,
|
|
"num_tokens": 13273278.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.3593845307826996,
|
|
"rewards/correctness_reward/mean": 0.225,
|
|
"rewards/correctness_reward/std": 0.3593845427036285,
|
|
"step": 2830,
|
|
"step_time": 23.790407656412572,
|
|
"zero_adv_skip_rate": 0.388
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 785.6,
|
|
"completions/max_terminated_length": 565.6,
|
|
"completions/mean_length": 263.9,
|
|
"completions/mean_terminated_length": 221.5714309692383,
|
|
"completions/min_length": 82.0,
|
|
"completions/min_terminated_length": 82.0,
|
|
"entropy": 0.34899605584463905,
|
|
"epoch": 0.37886872998932764,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.06376087665557861,
|
|
"num_tokens": 13321494.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.33288776576519014,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.3328877717256546,
|
|
"step": 2840,
|
|
"step_time": 21.341962857730685,
|
|
"zero_adv_skip_rate": 0.3877
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 531.9,
|
|
"completions/max_terminated_length": 449.0,
|
|
"completions/mean_length": 231.825,
|
|
"completions/mean_terminated_length": 210.53928909301757,
|
|
"completions/min_length": 89.9,
|
|
"completions/min_terminated_length": 89.9,
|
|
"entropy": 0.32765990202980383,
|
|
"epoch": 0.3802027748132337,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.007481733709573746,
|
|
"num_tokens": 13369080.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.2874556124210358,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.2874556124210358,
|
|
"step": 2850,
|
|
"step_time": 15.249559112917632,
|
|
"zero_adv_skip_rate": 0.3874
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 661.1,
|
|
"completions/max_terminated_length": 504.6,
|
|
"completions/mean_length": 254.9125,
|
|
"completions/mean_terminated_length": 232.5464309692383,
|
|
"completions/min_length": 90.5,
|
|
"completions/min_terminated_length": 90.5,
|
|
"entropy": 0.34401760119944813,
|
|
"epoch": 0.3815368196371398,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.83984375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.01775193065404892,
|
|
"num_tokens": 13421753.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 2860,
|
|
"step_time": 18.178027795441448,
|
|
"zero_adv_skip_rate": 0.3878
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 806.5,
|
|
"completions/max_terminated_length": 647.4,
|
|
"completions/mean_length": 301.95,
|
|
"completions/mean_terminated_length": 262.628577041626,
|
|
"completions/min_length": 83.9,
|
|
"completions/min_terminated_length": 83.9,
|
|
"entropy": 0.3806152530014515,
|
|
"epoch": 0.3828708644610459,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.03654001355171203,
|
|
"num_tokens": 13475037.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.14433692693710326,
|
|
"rewards/correctness_reward/mean": 0.1875,
|
|
"rewards/correctness_reward/std": 0.14433693289756774,
|
|
"step": 2870,
|
|
"step_time": 21.10724169909954,
|
|
"zero_adv_skip_rate": 0.3889
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 528.6,
|
|
"completions/max_terminated_length": 365.9,
|
|
"completions/mean_length": 212.1125,
|
|
"completions/mean_terminated_length": 185.9375015258789,
|
|
"completions/min_length": 98.2,
|
|
"completions/min_terminated_length": 98.2,
|
|
"entropy": 0.3321983404457569,
|
|
"epoch": 0.384204909284952,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0030149370431900024,
|
|
"num_tokens": 13518918.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.2756610929965973,
|
|
"rewards/correctness_reward/mean": 0.4375,
|
|
"rewards/correctness_reward/std": 0.27566109895706176,
|
|
"step": 2880,
|
|
"step_time": 14.750818557292224,
|
|
"zero_adv_skip_rate": 0.3889
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 267.1,
|
|
"completions/max_terminated_length": 265.2,
|
|
"completions/mean_length": 146.2375,
|
|
"completions/mean_terminated_length": 145.48928604125976,
|
|
"completions/min_length": 79.4,
|
|
"completions/min_terminated_length": 79.4,
|
|
"entropy": 0.37851950406496015,
|
|
"epoch": 0.3855389541088581,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.6171875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.030153191089630126,
|
|
"num_tokens": 13557873.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.36737640500068663,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.36737642884254457,
|
|
"step": 2890,
|
|
"step_time": 8.48622780656442,
|
|
"zero_adv_skip_rate": 0.3886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 869.8,
|
|
"completions/max_terminated_length": 508.2,
|
|
"completions/mean_length": 327.1375,
|
|
"completions/mean_terminated_length": 220.3375030517578,
|
|
"completions/min_length": 87.1,
|
|
"completions/min_terminated_length": 87.1,
|
|
"entropy": 0.3750612372532487,
|
|
"epoch": 0.38687299893276417,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.016105541586875917,
|
|
"num_tokens": 13612308.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.23144718110561371,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.23144719302654265,
|
|
"step": 2900,
|
|
"step_time": 23.061730061192065,
|
|
"zero_adv_skip_rate": 0.389
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 377.0,
|
|
"completions/max_terminated_length": 365.9,
|
|
"completions/mean_length": 176.2125,
|
|
"completions/mean_terminated_length": 174.01071472167968,
|
|
"completions/min_length": 81.5,
|
|
"completions/min_terminated_length": 81.5,
|
|
"entropy": 0.3847446172897305,
|
|
"epoch": 0.3882070437566702,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.54296875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.029777726531028746,
|
|
"num_tokens": 13656013.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.32402919232845306,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.32402920424938203,
|
|
"step": 2910,
|
|
"step_time": 11.238546008709818,
|
|
"zero_adv_skip_rate": 0.3887
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 480.4,
|
|
"completions/max_terminated_length": 469.0,
|
|
"completions/mean_length": 174.9875,
|
|
"completions/mean_terminated_length": 168.1732162475586,
|
|
"completions/min_length": 70.8,
|
|
"completions/min_terminated_length": 70.8,
|
|
"entropy": 0.2608511919776599,
|
|
"epoch": 0.3895410885805763,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.44140625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.014638954401016235,
|
|
"num_tokens": 13700236.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.2504029482603073,
|
|
"step": 2920,
|
|
"step_time": 13.493757304828614,
|
|
"zero_adv_skip_rate": 0.3887
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 429.4,
|
|
"completions/max_terminated_length": 427.9,
|
|
"completions/mean_length": 173.6875,
|
|
"completions/mean_terminated_length": 173.28035736083984,
|
|
"completions/min_length": 87.3,
|
|
"completions/min_terminated_length": 87.3,
|
|
"entropy": 0.3601234673988074,
|
|
"epoch": 0.3908751334044824,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.6484375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.02668473422527313,
|
|
"num_tokens": 13743059.0,
|
|
"reward": 0.4875,
|
|
"reward_std": 0.361081862449646,
|
|
"rewards/correctness_reward/mean": 0.4875,
|
|
"rewards/correctness_reward/std": 0.3610818684101105,
|
|
"step": 2930,
|
|
"step_time": 12.535552377533168,
|
|
"zero_adv_skip_rate": 0.3881
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 533.3,
|
|
"completions/max_terminated_length": 531.2,
|
|
"completions/mean_length": 209.5125,
|
|
"completions/mean_terminated_length": 208.27857208251953,
|
|
"completions/min_length": 76.1,
|
|
"completions/min_terminated_length": 76.1,
|
|
"entropy": 0.4305765531025827,
|
|
"epoch": 0.39220917822838847,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1247973084449768,
|
|
"num_tokens": 13798580.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.20325308442115783,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.2032530963420868,
|
|
"step": 2940,
|
|
"step_time": 14.780001030676067,
|
|
"zero_adv_skip_rate": 0.3888
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 677.1,
|
|
"completions/max_terminated_length": 358.7,
|
|
"completions/mean_length": 263.6625,
|
|
"completions/mean_terminated_length": 221.903572845459,
|
|
"completions/min_length": 133.9,
|
|
"completions/min_terminated_length": 133.9,
|
|
"entropy": 0.32004332058131696,
|
|
"epoch": 0.39354322305229456,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0018021494150161743,
|
|
"num_tokens": 13847217.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.23314451277256013,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.2331445187330246,
|
|
"step": 2950,
|
|
"step_time": 18.42562873773277,
|
|
"zero_adv_skip_rate": 0.3892
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 608.7,
|
|
"completions/max_terminated_length": 423.6,
|
|
"completions/mean_length": 208.025,
|
|
"completions/mean_terminated_length": 182.19821548461914,
|
|
"completions/min_length": 88.8,
|
|
"completions/min_terminated_length": 88.8,
|
|
"entropy": 0.40685561957070604,
|
|
"epoch": 0.39487726787620064,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.490234375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0063969030976295475,
|
|
"num_tokens": 13895395.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.35561010539531707,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.35561011731624603,
|
|
"step": 2960,
|
|
"step_time": 17.116681306343526,
|
|
"zero_adv_skip_rate": 0.3885
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 574.5,
|
|
"completions/max_terminated_length": 368.4,
|
|
"completions/mean_length": 213.475,
|
|
"completions/mean_terminated_length": 183.2875015258789,
|
|
"completions/min_length": 83.3,
|
|
"completions/min_terminated_length": 83.3,
|
|
"entropy": 0.3035817075786846,
|
|
"epoch": 0.39621131270010673,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.45703125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04093253016471863,
|
|
"num_tokens": 13940225.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.31479085683822633,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.3147908627986908,
|
|
"step": 2970,
|
|
"step_time": 16.1680516644381,
|
|
"zero_adv_skip_rate": 0.3882
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 633.6,
|
|
"completions/max_terminated_length": 509.2,
|
|
"completions/mean_length": 249.15,
|
|
"completions/mean_terminated_length": 228.77321548461913,
|
|
"completions/min_length": 89.6,
|
|
"completions/min_terminated_length": 89.6,
|
|
"entropy": 0.3198142705174784,
|
|
"epoch": 0.3975453575240128,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.50390625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.02488538920879364,
|
|
"num_tokens": 13989213.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.13509859144687653,
|
|
"rewards/correctness_reward/mean": 0.3375,
|
|
"rewards/correctness_reward/std": 0.13509859144687653,
|
|
"step": 2980,
|
|
"step_time": 16.871312510408462,
|
|
"zero_adv_skip_rate": 0.3893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 361.2,
|
|
"completions/max_terminated_length": 349.0,
|
|
"completions/mean_length": 163.35,
|
|
"completions/mean_terminated_length": 160.70178680419923,
|
|
"completions/min_length": 71.2,
|
|
"completions/min_terminated_length": 71.2,
|
|
"entropy": 0.4074404730151097,
|
|
"epoch": 0.3988794023479189,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.62890625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.004867422953248024,
|
|
"num_tokens": 14032153.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.28112500309944155,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.28112501502037046,
|
|
"step": 2990,
|
|
"step_time": 10.589477784186602,
|
|
"zero_adv_skip_rate": 0.3893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 561.4,
|
|
"completions/max_terminated_length": 418.7,
|
|
"completions/mean_length": 208.1125,
|
|
"completions/mean_terminated_length": 185.5357177734375,
|
|
"completions/min_length": 75.3,
|
|
"completions/min_terminated_length": 75.3,
|
|
"entropy": 0.3628857687115669,
|
|
"epoch": 0.400213447171825,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.002740319073200226,
|
|
"num_tokens": 14076306.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.22598327100276946,
|
|
"rewards/correctness_reward/mean": 0.5,
|
|
"rewards/correctness_reward/std": 0.22598327696323395,
|
|
"step": 3000,
|
|
"step_time": 15.43549137637019,
|
|
"zero_adv_skip_rate": 0.3897
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 441.9,
|
|
"completions/max_terminated_length": 434.4,
|
|
"completions/mean_length": 205.05,
|
|
"completions/mean_terminated_length": 201.9821434020996,
|
|
"completions/min_length": 81.5,
|
|
"completions/min_terminated_length": 81.5,
|
|
"entropy": 0.35479327384382486,
|
|
"epoch": 0.40154749199573103,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04314013719558716,
|
|
"num_tokens": 14122622.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.25840266346931456,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.2584026694297791,
|
|
"step": 3010,
|
|
"step_time": 12.539242709521204,
|
|
"zero_adv_skip_rate": 0.39
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 253.2,
|
|
"completions/max_terminated_length": 240.9,
|
|
"completions/mean_length": 123.4125,
|
|
"completions/mean_terminated_length": 120.86428604125976,
|
|
"completions/min_length": 59.8,
|
|
"completions/min_terminated_length": 59.8,
|
|
"entropy": 0.3716567654162645,
|
|
"epoch": 0.4028815368196371,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.49609375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.10521469116210938,
|
|
"num_tokens": 14159647.0,
|
|
"reward": 0.5375,
|
|
"reward_std": 0.43179253935813905,
|
|
"rewards/correctness_reward/mean": 0.5375,
|
|
"rewards/correctness_reward/std": 0.4317925453186035,
|
|
"step": 3020,
|
|
"step_time": 8.376569910906255,
|
|
"zero_adv_skip_rate": 0.3887
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 499.2,
|
|
"completions/max_terminated_length": 499.2,
|
|
"completions/mean_length": 206.5625,
|
|
"completions/mean_terminated_length": 206.5625,
|
|
"completions/min_length": 81.1,
|
|
"completions/min_terminated_length": 81.1,
|
|
"entropy": 0.38780603744089603,
|
|
"epoch": 0.4042155816435432,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.12801626920700074,
|
|
"num_tokens": 14205308.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.23691109120845794,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.23691110908985138,
|
|
"step": 3030,
|
|
"step_time": 13.827406360581517,
|
|
"zero_adv_skip_rate": 0.3891
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 544.6,
|
|
"completions/max_terminated_length": 450.1,
|
|
"completions/mean_length": 228.7875,
|
|
"completions/mean_terminated_length": 208.12142944335938,
|
|
"completions/min_length": 102.4,
|
|
"completions/min_terminated_length": 102.4,
|
|
"entropy": 0.3478173855692148,
|
|
"epoch": 0.4055496264674493,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.6484375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.02181261032819748,
|
|
"num_tokens": 14250403.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.1595182627439499,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.1595182627439499,
|
|
"step": 3040,
|
|
"step_time": 14.916030063014478,
|
|
"zero_adv_skip_rate": 0.3898
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 291.6,
|
|
"completions/max_terminated_length": 291.1,
|
|
"completions/mean_length": 167.275,
|
|
"completions/mean_terminated_length": 166.49285812377929,
|
|
"completions/min_length": 72.9,
|
|
"completions/min_terminated_length": 72.9,
|
|
"entropy": 0.31652401605000097,
|
|
"epoch": 0.4068836712913554,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.59375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0493713915348053,
|
|
"num_tokens": 14290393.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.2558668524026871,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.25586686432361605,
|
|
"step": 3050,
|
|
"step_time": 8.87832219451666,
|
|
"zero_adv_skip_rate": 0.3898
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 594.1,
|
|
"completions/max_terminated_length": 594.0,
|
|
"completions/mean_length": 257.375,
|
|
"completions/mean_terminated_length": 257.1214286804199,
|
|
"completions/min_length": 114.4,
|
|
"completions/min_terminated_length": 114.4,
|
|
"entropy": 0.3143807068201048,
|
|
"epoch": 0.4082177161152615,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1836953043937683,
|
|
"num_tokens": 14340335.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.29839127957820893,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.29839127957820893,
|
|
"step": 3060,
|
|
"step_time": 16.752322817221284,
|
|
"zero_adv_skip_rate": 0.3895
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 617.4,
|
|
"completions/max_terminated_length": 435.8,
|
|
"completions/mean_length": 205.875,
|
|
"completions/mean_terminated_length": 182.0678581237793,
|
|
"completions/min_length": 80.3,
|
|
"completions/min_terminated_length": 80.3,
|
|
"entropy": 0.3935362044721842,
|
|
"epoch": 0.40955176093916756,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.07340641617774964,
|
|
"num_tokens": 14387845.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.2874556124210358,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.2874556124210358,
|
|
"step": 3070,
|
|
"step_time": 17.360236372146755,
|
|
"zero_adv_skip_rate": 0.3893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 498.7,
|
|
"completions/max_terminated_length": 317.6,
|
|
"completions/mean_length": 179.725,
|
|
"completions/mean_terminated_length": 154.27321548461913,
|
|
"completions/min_length": 68.1,
|
|
"completions/min_terminated_length": 68.1,
|
|
"entropy": 0.3614945448935032,
|
|
"epoch": 0.41088580576307365,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.015516254305839538,
|
|
"num_tokens": 14430431.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.2857582807540894,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.28575828671455383,
|
|
"step": 3080,
|
|
"step_time": 14.512255771644414,
|
|
"zero_adv_skip_rate": 0.389
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 742.0,
|
|
"completions/max_terminated_length": 473.8,
|
|
"completions/mean_length": 284.5625,
|
|
"completions/mean_terminated_length": 242.5785774230957,
|
|
"completions/min_length": 108.1,
|
|
"completions/min_terminated_length": 108.1,
|
|
"entropy": 0.25109710562974213,
|
|
"epoch": 0.41221985058697974,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.05886385440826416,
|
|
"num_tokens": 14482092.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.19864802658557892,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.19864802658557892,
|
|
"step": 3090,
|
|
"step_time": 20.861568889953197,
|
|
"zero_adv_skip_rate": 0.3893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 354.5,
|
|
"completions/max_terminated_length": 354.5,
|
|
"completions/mean_length": 179.425,
|
|
"completions/mean_terminated_length": 179.425,
|
|
"completions/min_length": 72.5,
|
|
"completions/min_terminated_length": 72.5,
|
|
"entropy": 0.4314347955708702,
|
|
"epoch": 0.41355389541088583,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.80859375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.042150062322616574,
|
|
"num_tokens": 14527254.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.2886738538742065,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.2886738657951355,
|
|
"step": 3100,
|
|
"step_time": 10.885504367388785,
|
|
"zero_adv_skip_rate": 0.3894
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 786.3,
|
|
"completions/max_terminated_length": 262.5,
|
|
"completions/mean_length": 246.2625,
|
|
"completions/mean_terminated_length": 151.44643173217773,
|
|
"completions/min_length": 71.6,
|
|
"completions/min_terminated_length": 71.6,
|
|
"entropy": 0.38380289264023304,
|
|
"epoch": 0.41488794023479186,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.023955520987510682,
|
|
"num_tokens": 14576355.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.17422835528850555,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.17422835528850555,
|
|
"step": 3110,
|
|
"step_time": 21.38902228930965,
|
|
"zero_adv_skip_rate": 0.39
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 710.5,
|
|
"completions/max_terminated_length": 528.2,
|
|
"completions/mean_length": 220.325,
|
|
"completions/mean_terminated_length": 194.88928756713867,
|
|
"completions/min_length": 70.9,
|
|
"completions/min_terminated_length": 70.9,
|
|
"entropy": 0.3940169561654329,
|
|
"epoch": 0.41622198505869795,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6328125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.01643856167793274,
|
|
"num_tokens": 14624557.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.22051936089992524,
|
|
"rewards/correctness_reward/mean": 0.2375,
|
|
"rewards/correctness_reward/std": 0.22051936089992524,
|
|
"step": 3120,
|
|
"step_time": 19.42443821989,
|
|
"zero_adv_skip_rate": 0.3904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 564.7,
|
|
"completions/max_terminated_length": 364.9,
|
|
"completions/mean_length": 232.075,
|
|
"completions/mean_terminated_length": 180.38988151550294,
|
|
"completions/min_length": 64.8,
|
|
"completions/min_terminated_length": 64.8,
|
|
"entropy": 0.37052817828953266,
|
|
"epoch": 0.41755602988260404,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.392578125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04394925236701965,
|
|
"num_tokens": 14674267.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.27397160828113554,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.27397160828113554,
|
|
"step": 3130,
|
|
"step_time": 15.826927597634494,
|
|
"zero_adv_skip_rate": 0.3904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 832.1,
|
|
"completions/max_terminated_length": 643.3,
|
|
"completions/mean_length": 271.2125,
|
|
"completions/mean_terminated_length": 246.21071548461913,
|
|
"completions/min_length": 80.6,
|
|
"completions/min_terminated_length": 80.6,
|
|
"entropy": 0.3855565955551962,
|
|
"epoch": 0.41889007470651013,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.447265625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.11559396982192993,
|
|
"num_tokens": 14724660.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.28489942848682404,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.284899440407753,
|
|
"step": 3140,
|
|
"step_time": 22.88993926560506,
|
|
"zero_adv_skip_rate": 0.3904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 704.7,
|
|
"completions/max_terminated_length": 540.8,
|
|
"completions/mean_length": 293.2875,
|
|
"completions/mean_terminated_length": 246.19940643310548,
|
|
"completions/min_length": 105.7,
|
|
"completions/min_terminated_length": 105.7,
|
|
"entropy": 0.4372224261363347,
|
|
"epoch": 0.4202241195304162,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.046436816453933716,
|
|
"num_tokens": 14775163.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.14433692693710326,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.14433693289756774,
|
|
"step": 3150,
|
|
"step_time": 19.00913180662319,
|
|
"zero_adv_skip_rate": 0.3914
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 498.8,
|
|
"completions/max_terminated_length": 310.9,
|
|
"completions/mean_length": 181.25,
|
|
"completions/mean_terminated_length": 156.716072845459,
|
|
"completions/min_length": 86.9,
|
|
"completions/min_terminated_length": 86.9,
|
|
"entropy": 0.36184400473056094,
|
|
"epoch": 0.4215581643543223,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.7734375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.09931734800338746,
|
|
"num_tokens": 14817919.0,
|
|
"reward": 0.5125,
|
|
"reward_std": 0.31855743527412417,
|
|
"rewards/correctness_reward/mean": 0.5125,
|
|
"rewards/correctness_reward/std": 0.3185574531555176,
|
|
"step": 3160,
|
|
"step_time": 14.393841470312328,
|
|
"zero_adv_skip_rate": 0.3911
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 507.1,
|
|
"completions/max_terminated_length": 439.9,
|
|
"completions/mean_length": 262.1625,
|
|
"completions/mean_terminated_length": 244.05714645385743,
|
|
"completions/min_length": 114.2,
|
|
"completions/min_terminated_length": 114.2,
|
|
"entropy": 0.4277646791888401,
|
|
"epoch": 0.4228922091782284,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.2412109375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.028056353330612183,
|
|
"num_tokens": 14867452.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.332049286365509,
|
|
"rewards/correctness_reward/mean": 0.2375,
|
|
"rewards/correctness_reward/std": 0.3320492923259735,
|
|
"step": 3170,
|
|
"step_time": 14.776594763994217,
|
|
"zero_adv_skip_rate": 0.3905
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 363.9,
|
|
"completions/max_terminated_length": 363.9,
|
|
"completions/mean_length": 178.6375,
|
|
"completions/mean_terminated_length": 178.27678604125975,
|
|
"completions/min_length": 71.6,
|
|
"completions/min_terminated_length": 71.6,
|
|
"entropy": 0.3734993330602135,
|
|
"epoch": 0.4242262540021345,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.482421875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.020479395985603333,
|
|
"num_tokens": 14909575.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.3038551896810532,
|
|
"rewards/correctness_reward/mean": 0.5,
|
|
"rewards/correctness_reward/std": 0.30385519564151764,
|
|
"step": 3180,
|
|
"step_time": 10.961421271599828,
|
|
"zero_adv_skip_rate": 0.3903
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 577.2,
|
|
"completions/max_terminated_length": 575.3,
|
|
"completions/mean_length": 245.0125,
|
|
"completions/mean_terminated_length": 243.88928756713867,
|
|
"completions/min_length": 83.2,
|
|
"completions/min_terminated_length": 83.2,
|
|
"entropy": 0.4305568277569754,
|
|
"epoch": 0.4255602988260406,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.13565030097961425,
|
|
"num_tokens": 14959160.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.3257186770439148,
|
|
"rewards/correctness_reward/mean": 0.4375,
|
|
"rewards/correctness_reward/std": 0.32571869492530825,
|
|
"step": 3190,
|
|
"step_time": 16.34901381796226,
|
|
"zero_adv_skip_rate": 0.39
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 255.1,
|
|
"completions/max_terminated_length": 255.1,
|
|
"completions/mean_length": 138.0375,
|
|
"completions/mean_terminated_length": 138.0375,
|
|
"completions/min_length": 71.0,
|
|
"completions/min_terminated_length": 71.0,
|
|
"entropy": 0.3304637718413557,
|
|
"epoch": 0.42689434364994666,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.52734375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04635384678840637,
|
|
"num_tokens": 14996067.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.33458509743213655,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.33458509743213655,
|
|
"step": 3200,
|
|
"step_time": 8.140461628418416,
|
|
"zero_adv_skip_rate": 0.3897
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 630.5,
|
|
"completions/max_terminated_length": 451.3,
|
|
"completions/mean_length": 222.7,
|
|
"completions/mean_terminated_length": 197.71785736083984,
|
|
"completions/min_length": 73.9,
|
|
"completions/min_terminated_length": 73.9,
|
|
"entropy": 0.4034978337585926,
|
|
"epoch": 0.4282283884738527,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.6484375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.1062925934791565,
|
|
"num_tokens": 15042339.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.26849985122680664,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.2684998571872711,
|
|
"step": 3210,
|
|
"step_time": 17.329608189407736,
|
|
"zero_adv_skip_rate": 0.3897
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 808.0,
|
|
"completions/max_terminated_length": 616.0,
|
|
"completions/mean_length": 254.7875,
|
|
"completions/mean_terminated_length": 212.02143249511718,
|
|
"completions/min_length": 89.3,
|
|
"completions/min_terminated_length": 89.3,
|
|
"entropy": 0.3667602820787579,
|
|
"epoch": 0.4295624332977588,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.384765625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.019642768800258635,
|
|
"num_tokens": 15089882.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.3130935251712799,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.3130935370922089,
|
|
"step": 3220,
|
|
"step_time": 21.962425818201154,
|
|
"zero_adv_skip_rate": 0.3894
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 632.2,
|
|
"completions/max_terminated_length": 613.0,
|
|
"completions/mean_length": 273.15,
|
|
"completions/mean_terminated_length": 270.14107208251954,
|
|
"completions/min_length": 105.7,
|
|
"completions/min_terminated_length": 105.7,
|
|
"entropy": 0.39195155367876094,
|
|
"epoch": 0.43089647812166487,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.423828125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.012867695093154908,
|
|
"num_tokens": 15145686.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/correctness_reward/mean": 0.225,
|
|
"rewards/correctness_reward/std": 0.2504029482603073,
|
|
"step": 3230,
|
|
"step_time": 17.506435258220883,
|
|
"zero_adv_skip_rate": 0.3895
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 289.1,
|
|
"completions/max_terminated_length": 288.2,
|
|
"completions/mean_length": 156.1375,
|
|
"completions/mean_terminated_length": 155.4571434020996,
|
|
"completions/min_length": 86.4,
|
|
"completions/min_terminated_length": 86.4,
|
|
"entropy": 0.3709902656264603,
|
|
"epoch": 0.43223052294557096,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.546875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0028678208589553835,
|
|
"num_tokens": 15187801.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.18138959705829621,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.18138959705829621,
|
|
"step": 3240,
|
|
"step_time": 8.516131642553955,
|
|
"zero_adv_skip_rate": 0.3901
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 311.5,
|
|
"completions/max_terminated_length": 311.5,
|
|
"completions/mean_length": 160.8125,
|
|
"completions/mean_terminated_length": 160.8125,
|
|
"completions/min_length": 90.8,
|
|
"completions/min_terminated_length": 90.8,
|
|
"entropy": 0.32833857954080614,
|
|
"epoch": 0.43356456776947705,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04587138295173645,
|
|
"num_tokens": 15229898.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.3202547669410706,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.3202547788619995,
|
|
"step": 3250,
|
|
"step_time": 9.598452550079674,
|
|
"zero_adv_skip_rate": 0.3898
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 427.5,
|
|
"completions/max_terminated_length": 417.9,
|
|
"completions/mean_length": 220.9375,
|
|
"completions/mean_terminated_length": 219.48750038146972,
|
|
"completions/min_length": 80.1,
|
|
"completions/min_terminated_length": 80.1,
|
|
"entropy": 0.3985643535852432,
|
|
"epoch": 0.43489861259338314,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.03550328612327576,
|
|
"num_tokens": 15278125.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.2112731784582138,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.2112731844186783,
|
|
"step": 3260,
|
|
"step_time": 12.289964746218175,
|
|
"zero_adv_skip_rate": 0.3902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 577.0,
|
|
"completions/max_terminated_length": 574.6,
|
|
"completions/mean_length": 240.2125,
|
|
"completions/mean_terminated_length": 239.5607147216797,
|
|
"completions/min_length": 83.1,
|
|
"completions/min_terminated_length": 83.1,
|
|
"entropy": 0.45053806751966474,
|
|
"epoch": 0.4362326574172892,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 1.453125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04338962733745575,
|
|
"num_tokens": 15324854.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.23691109120845794,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.23691110908985138,
|
|
"step": 3270,
|
|
"step_time": 16.34690610151738,
|
|
"zero_adv_skip_rate": 0.3905
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 469.5,
|
|
"completions/max_terminated_length": 422.5,
|
|
"completions/mean_length": 251.025,
|
|
"completions/mean_terminated_length": 222.4678581237793,
|
|
"completions/min_length": 95.7,
|
|
"completions/min_terminated_length": 95.7,
|
|
"entropy": 0.4414745173417032,
|
|
"epoch": 0.4375667022411953,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.59765625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04631409347057343,
|
|
"num_tokens": 15372680.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.33921052813529967,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.3392105340957642,
|
|
"step": 3280,
|
|
"step_time": 13.784434454515576,
|
|
"zero_adv_skip_rate": 0.3899
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 625.2,
|
|
"completions/max_terminated_length": 293.9,
|
|
"completions/mean_length": 212.0125,
|
|
"completions/mean_terminated_length": 163.50000381469727,
|
|
"completions/min_length": 72.0,
|
|
"completions/min_terminated_length": 72.0,
|
|
"entropy": 0.4548579044640064,
|
|
"epoch": 0.4389007470651014,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.06590721607208253,
|
|
"num_tokens": 15419841.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.1632926881313324,
|
|
"rewards/correctness_reward/mean": 0.175,
|
|
"rewards/correctness_reward/std": 0.1632926881313324,
|
|
"step": 3290,
|
|
"step_time": 16.985018668323754,
|
|
"zero_adv_skip_rate": 0.3906
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 473.4,
|
|
"completions/max_terminated_length": 401.0,
|
|
"completions/mean_length": 192.775,
|
|
"completions/mean_terminated_length": 172.6714309692383,
|
|
"completions/min_length": 74.0,
|
|
"completions/min_terminated_length": 74.0,
|
|
"entropy": 0.37901414558291435,
|
|
"epoch": 0.4402347918890075,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.037241813540458676,
|
|
"num_tokens": 15464327.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.2722742766141891,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.27227428257465364,
|
|
"step": 3300,
|
|
"step_time": 13.490771828312427,
|
|
"zero_adv_skip_rate": 0.3906
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 974.0,
|
|
"completions/max_terminated_length": 510.5,
|
|
"completions/mean_length": 330.6125,
|
|
"completions/mean_terminated_length": 265.89286346435546,
|
|
"completions/min_length": 133.8,
|
|
"completions/min_terminated_length": 133.8,
|
|
"entropy": 0.3653562720865011,
|
|
"epoch": 0.4415688367129136,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.09767056107521058,
|
|
"num_tokens": 15520880.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.1977891743183136,
|
|
"rewards/correctness_reward/mean": 0.1875,
|
|
"rewards/correctness_reward/std": 0.19778918027877807,
|
|
"step": 3310,
|
|
"step_time": 26.732110422197728,
|
|
"zero_adv_skip_rate": 0.3912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 694.3,
|
|
"completions/max_terminated_length": 586.2,
|
|
"completions/mean_length": 293.575,
|
|
"completions/mean_terminated_length": 273.5928588867188,
|
|
"completions/min_length": 88.9,
|
|
"completions/min_terminated_length": 88.9,
|
|
"entropy": 0.42176809534430504,
|
|
"epoch": 0.4429028815368196,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.41796875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.003262786567211151,
|
|
"num_tokens": 15573806.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.13340125977993011,
|
|
"rewards/correctness_reward/mean": 0.125,
|
|
"rewards/correctness_reward/std": 0.1334012657403946,
|
|
"step": 3320,
|
|
"step_time": 19.878994594421236,
|
|
"zero_adv_skip_rate": 0.3922
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 497.8,
|
|
"completions/max_terminated_length": 493.3,
|
|
"completions/mean_length": 204.75,
|
|
"completions/mean_terminated_length": 203.8357147216797,
|
|
"completions/min_length": 79.3,
|
|
"completions/min_terminated_length": 79.3,
|
|
"entropy": 0.37879737094044685,
|
|
"epoch": 0.4442369263607257,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.13934110403060912,
|
|
"num_tokens": 15618898.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.23484184443950654,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.23484184443950654,
|
|
"step": 3330,
|
|
"step_time": 14.878186760656536,
|
|
"zero_adv_skip_rate": 0.3925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 657.4,
|
|
"completions/max_terminated_length": 505.6,
|
|
"completions/mean_length": 247.0,
|
|
"completions/mean_terminated_length": 225.07678833007813,
|
|
"completions/min_length": 94.1,
|
|
"completions/min_terminated_length": 94.1,
|
|
"entropy": 0.29702246685822803,
|
|
"epoch": 0.4455709711846318,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.048518973588943484,
|
|
"num_tokens": 15668962.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.28489942848682404,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.284899440407753,
|
|
"step": 3340,
|
|
"step_time": 19.179389284178615,
|
|
"zero_adv_skip_rate": 0.3925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 538.7,
|
|
"completions/max_terminated_length": 331.5,
|
|
"completions/mean_length": 179.85,
|
|
"completions/mean_terminated_length": 153.44285774230957,
|
|
"completions/min_length": 75.7,
|
|
"completions/min_terminated_length": 75.7,
|
|
"entropy": 0.3922895509749651,
|
|
"epoch": 0.4469050160085379,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 1.3046875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.031350824236869815,
|
|
"num_tokens": 15710214.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.18771235942840575,
|
|
"rewards/correctness_reward/mean": 0.45,
|
|
"rewards/correctness_reward/std": 0.18771235942840575,
|
|
"step": 3350,
|
|
"step_time": 15.835942746698857,
|
|
"zero_adv_skip_rate": 0.3928
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 359.4,
|
|
"completions/max_terminated_length": 346.5,
|
|
"completions/mean_length": 154.7875,
|
|
"completions/mean_terminated_length": 152.03214416503906,
|
|
"completions/min_length": 77.0,
|
|
"completions/min_terminated_length": 77.0,
|
|
"entropy": 0.4105710467944543,
|
|
"epoch": 0.44823906083244397,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.48828125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.014115083217620849,
|
|
"num_tokens": 15752253.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.28659676015377045,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.2865967661142349,
|
|
"step": 3360,
|
|
"step_time": 11.257349693309516,
|
|
"zero_adv_skip_rate": 0.3929
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 588.3,
|
|
"completions/max_terminated_length": 427.8,
|
|
"completions/mean_length": 212.5875,
|
|
"completions/mean_terminated_length": 186.2303596496582,
|
|
"completions/min_length": 75.7,
|
|
"completions/min_terminated_length": 75.7,
|
|
"entropy": 0.2911765268072486,
|
|
"epoch": 0.44957310565635006,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.05335766077041626,
|
|
"num_tokens": 15799020.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.3375,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 3370,
|
|
"step_time": 17.225827161874623,
|
|
"zero_adv_skip_rate": 0.3932
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 455.4,
|
|
"completions/max_terminated_length": 429.9,
|
|
"completions/mean_length": 222.9125,
|
|
"completions/mean_terminated_length": 218.6214298248291,
|
|
"completions/min_length": 97.8,
|
|
"completions/min_terminated_length": 97.8,
|
|
"entropy": 0.43690106049180033,
|
|
"epoch": 0.45090715048025615,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.01767892241477966,
|
|
"num_tokens": 15845125.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.23691109120845794,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.23691110908985138,
|
|
"step": 3380,
|
|
"step_time": 13.934144701808691,
|
|
"zero_adv_skip_rate": 0.3935
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 181.1,
|
|
"completions/max_terminated_length": 167.9,
|
|
"completions/mean_length": 108.15,
|
|
"completions/mean_terminated_length": 105.24285888671875,
|
|
"completions/min_length": 57.4,
|
|
"completions/min_terminated_length": 57.4,
|
|
"entropy": 0.36406522849574685,
|
|
"epoch": 0.45224119530416224,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.03747690916061401,
|
|
"num_tokens": 15881353.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.3610740154981613,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.36107403337955474,
|
|
"step": 3390,
|
|
"step_time": 6.857184922322631,
|
|
"zero_adv_skip_rate": 0.3929
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 627.8,
|
|
"completions/max_terminated_length": 381.0,
|
|
"completions/mean_length": 249.925,
|
|
"completions/mean_terminated_length": 207.08393325805665,
|
|
"completions/min_length": 96.4,
|
|
"completions/min_terminated_length": 96.4,
|
|
"entropy": 0.38543313803772133,
|
|
"epoch": 0.4535752401280683,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.007006685435771942,
|
|
"num_tokens": 15928187.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.12793734967708587,
|
|
"rewards/correctness_reward/mean": 0.3375,
|
|
"rewards/correctness_reward/std": 0.12793734967708587,
|
|
"step": 3400,
|
|
"step_time": 17.79933924814686,
|
|
"zero_adv_skip_rate": 0.3938
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 495.3,
|
|
"completions/max_terminated_length": 467.1,
|
|
"completions/mean_length": 221.1375,
|
|
"completions/mean_terminated_length": 201.48928833007812,
|
|
"completions/min_length": 72.0,
|
|
"completions/min_terminated_length": 72.0,
|
|
"entropy": 0.38095253985375166,
|
|
"epoch": 0.4549092849519744,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.58984375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04590202867984772,
|
|
"num_tokens": 15973390.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.361081862449646,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.3610818684101105,
|
|
"step": 3410,
|
|
"step_time": 15.813159693218768,
|
|
"zero_adv_skip_rate": 0.3933
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 832.4,
|
|
"completions/max_terminated_length": 524.9,
|
|
"completions/mean_length": 299.825,
|
|
"completions/mean_terminated_length": 252.8839324951172,
|
|
"completions/min_length": 95.2,
|
|
"completions/min_terminated_length": 95.2,
|
|
"entropy": 0.42291568658713785,
|
|
"epoch": 0.45624332977588045,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.006469470262527466,
|
|
"num_tokens": 16029856.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.3257186770439148,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.32571869492530825,
|
|
"step": 3420,
|
|
"step_time": 24.519353307876735,
|
|
"zero_adv_skip_rate": 0.393
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 826.5,
|
|
"completions/max_terminated_length": 779.8,
|
|
"completions/mean_length": 317.4375,
|
|
"completions/mean_terminated_length": 301.0035766601562,
|
|
"completions/min_length": 104.2,
|
|
"completions/min_terminated_length": 104.2,
|
|
"entropy": 0.3181531595598374,
|
|
"epoch": 0.45757737459978653,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.02037317156791687,
|
|
"num_tokens": 16083947.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.35305392146110537,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.35305394530296325,
|
|
"step": 3430,
|
|
"step_time": 24.767858824599536,
|
|
"zero_adv_skip_rate": 0.3927
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 539.5,
|
|
"completions/max_terminated_length": 325.8,
|
|
"completions/mean_length": 185.6,
|
|
"completions/mean_terminated_length": 154.80893173217774,
|
|
"completions/min_length": 78.0,
|
|
"completions/min_terminated_length": 78.0,
|
|
"entropy": 0.364984939340502,
|
|
"epoch": 0.4589114194236926,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.240234375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.012001962214708329,
|
|
"num_tokens": 16126419.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.3265775293111801,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.3265775412321091,
|
|
"step": 3440,
|
|
"step_time": 17.406816693302243,
|
|
"zero_adv_skip_rate": 0.3922
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 678.1,
|
|
"completions/max_terminated_length": 492.5,
|
|
"completions/mean_length": 250.45,
|
|
"completions/mean_terminated_length": 226.49285774230958,
|
|
"completions/min_length": 97.8,
|
|
"completions/min_terminated_length": 97.8,
|
|
"entropy": 0.3572706701233983,
|
|
"epoch": 0.4602454642475987,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04112858772277832,
|
|
"num_tokens": 16178015.0,
|
|
"reward": 0.475,
|
|
"reward_std": 0.24030575454235076,
|
|
"rewards/correctness_reward/mean": 0.475,
|
|
"rewards/correctness_reward/std": 0.24030576050281524,
|
|
"step": 3450,
|
|
"step_time": 20.278545079659672,
|
|
"zero_adv_skip_rate": 0.3925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 779.1,
|
|
"completions/max_terminated_length": 478.1,
|
|
"completions/mean_length": 287.8,
|
|
"completions/mean_terminated_length": 216.85833663940429,
|
|
"completions/min_length": 89.9,
|
|
"completions/min_terminated_length": 89.9,
|
|
"entropy": 0.40199789265170693,
|
|
"epoch": 0.4615795090715048,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.00785471946001053,
|
|
"num_tokens": 16228463.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.26680251955986023,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.2668025314807892,
|
|
"step": 3460,
|
|
"step_time": 23.271696721576156,
|
|
"zero_adv_skip_rate": 0.3925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 792.3,
|
|
"completions/max_terminated_length": 314.5,
|
|
"completions/mean_length": 224.8625,
|
|
"completions/mean_terminated_length": 158.2892868041992,
|
|
"completions/min_length": 81.2,
|
|
"completions/min_terminated_length": 81.2,
|
|
"entropy": 0.30652952641248704,
|
|
"epoch": 0.4629135538954109,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.05741763710975647,
|
|
"num_tokens": 16277052.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.22598327100276946,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.22598327696323395,
|
|
"step": 3470,
|
|
"step_time": 23.628059020638467,
|
|
"zero_adv_skip_rate": 0.3928
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 329.0,
|
|
"completions/max_terminated_length": 316.8,
|
|
"completions/mean_length": 178.5875,
|
|
"completions/mean_terminated_length": 170.05893173217774,
|
|
"completions/min_length": 102.0,
|
|
"completions/min_terminated_length": 102.0,
|
|
"entropy": 0.34843511579351294,
|
|
"epoch": 0.464247598719317,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.263671875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0827571451663971,
|
|
"num_tokens": 16320755.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.3909654438495636,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.3909654557704926,
|
|
"step": 3480,
|
|
"step_time": 11.225054739043117,
|
|
"zero_adv_skip_rate": 0.392
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 681.9,
|
|
"completions/max_terminated_length": 509.7,
|
|
"completions/mean_length": 264.1875,
|
|
"completions/mean_terminated_length": 241.48928680419922,
|
|
"completions/min_length": 117.6,
|
|
"completions/min_terminated_length": 117.6,
|
|
"entropy": 0.28562078438699245,
|
|
"epoch": 0.46558164354322307,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.005127435922622681,
|
|
"num_tokens": 16371226.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.26681036651134493,
|
|
"rewards/correctness_reward/mean": 0.2375,
|
|
"rewards/correctness_reward/std": 0.26681036651134493,
|
|
"step": 3490,
|
|
"step_time": 20.410283180419356,
|
|
"zero_adv_skip_rate": 0.392
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 228.0,
|
|
"completions/max_terminated_length": 225.4,
|
|
"completions/mean_length": 129.525,
|
|
"completions/mean_terminated_length": 128.35000076293946,
|
|
"completions/min_length": 68.7,
|
|
"completions/min_terminated_length": 68.7,
|
|
"entropy": 0.3527666690060869,
|
|
"epoch": 0.46691568836712916,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.009477762132883072,
|
|
"num_tokens": 16409172.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.35013834834098817,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.3501383662223816,
|
|
"step": 3500,
|
|
"step_time": 8.176886513922364,
|
|
"zero_adv_skip_rate": 0.3914
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 601.2,
|
|
"completions/max_terminated_length": 317.2,
|
|
"completions/mean_length": 239.8125,
|
|
"completions/mean_terminated_length": 174.74524536132813,
|
|
"completions/min_length": 97.7,
|
|
"completions/min_terminated_length": 97.7,
|
|
"entropy": 0.43417290567109984,
|
|
"epoch": 0.46824973319103524,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1316324234008789,
|
|
"num_tokens": 16456885.0,
|
|
"reward": 0.475,
|
|
"reward_std": 0.26133076250553133,
|
|
"rewards/correctness_reward/mean": 0.475,
|
|
"rewards/correctness_reward/std": 0.26133078038692475,
|
|
"step": 3510,
|
|
"step_time": 18.161609594523906,
|
|
"zero_adv_skip_rate": 0.3915
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 643.6,
|
|
"completions/max_terminated_length": 479.4,
|
|
"completions/mean_length": 255.1875,
|
|
"completions/mean_terminated_length": 228.7107162475586,
|
|
"completions/min_length": 96.4,
|
|
"completions/min_terminated_length": 96.4,
|
|
"entropy": 0.34939822293817996,
|
|
"epoch": 0.4695837780149413,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6796875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.11626236438751221,
|
|
"num_tokens": 16507860.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.18771235942840575,
|
|
"rewards/correctness_reward/mean": 0.075,
|
|
"rewards/correctness_reward/std": 0.18771235942840575,
|
|
"step": 3520,
|
|
"step_time": 18.9630541828461,
|
|
"zero_adv_skip_rate": 0.3918
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 536.8,
|
|
"completions/max_terminated_length": 371.2,
|
|
"completions/mean_length": 234.85,
|
|
"completions/mean_terminated_length": 188.4077392578125,
|
|
"completions/min_length": 78.3,
|
|
"completions/min_terminated_length": 78.3,
|
|
"entropy": 0.3329969147220254,
|
|
"epoch": 0.47091782283884737,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0015343319624662398,
|
|
"num_tokens": 16557680.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.22768060266971588,
|
|
"rewards/correctness_reward/mean": 0.1875,
|
|
"rewards/correctness_reward/std": 0.22768060266971588,
|
|
"step": 3530,
|
|
"step_time": 16.117773772682995,
|
|
"zero_adv_skip_rate": 0.3921
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 417.0,
|
|
"completions/max_terminated_length": 408.6,
|
|
"completions/mean_length": 206.1375,
|
|
"completions/mean_terminated_length": 199.93750228881837,
|
|
"completions/min_length": 89.3,
|
|
"completions/min_terminated_length": 89.3,
|
|
"entropy": 0.3996412018314004,
|
|
"epoch": 0.47225186766275346,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.36328125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04348827600479126,
|
|
"num_tokens": 16604915.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.24662851691246032,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.2466285228729248,
|
|
"step": 3540,
|
|
"step_time": 12.985562540683896,
|
|
"zero_adv_skip_rate": 0.3921
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 531.8,
|
|
"completions/max_terminated_length": 417.9,
|
|
"completions/mean_length": 225.4875,
|
|
"completions/mean_terminated_length": 204.8250026702881,
|
|
"completions/min_length": 77.1,
|
|
"completions/min_terminated_length": 77.1,
|
|
"entropy": 0.43328591994941235,
|
|
"epoch": 0.47358591248665954,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.515625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.020556829869747162,
|
|
"num_tokens": 16649298.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.18138959705829621,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.18138959705829621,
|
|
"step": 3550,
|
|
"step_time": 15.733971172850579,
|
|
"zero_adv_skip_rate": 0.3927
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 445.2,
|
|
"completions/max_terminated_length": 279.9,
|
|
"completions/mean_length": 198.575,
|
|
"completions/mean_terminated_length": 175.56964416503905,
|
|
"completions/min_length": 77.9,
|
|
"completions/min_terminated_length": 77.9,
|
|
"entropy": 0.3109995752040829,
|
|
"epoch": 0.47491995731056563,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.734375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.008282551914453507,
|
|
"num_tokens": 16692576.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.34211825430393217,
|
|
"rewards/correctness_reward/mean": 0.4375,
|
|
"rewards/correctness_reward/std": 0.3421182781457901,
|
|
"step": 3560,
|
|
"step_time": 13.838000807259231,
|
|
"zero_adv_skip_rate": 0.3924
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 275.9,
|
|
"completions/max_terminated_length": 249.6,
|
|
"completions/mean_length": 138.8625,
|
|
"completions/mean_terminated_length": 134.33035888671876,
|
|
"completions/min_length": 68.8,
|
|
"completions/min_terminated_length": 68.8,
|
|
"entropy": 0.578465369840463,
|
|
"epoch": 0.4762540021344717,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.65625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.019043627381324767,
|
|
"num_tokens": 16734477.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.15149816870689392,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.1514981746673584,
|
|
"step": 3570,
|
|
"step_time": 8.652484191209078,
|
|
"zero_adv_skip_rate": 0.3933
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 583.3,
|
|
"completions/max_terminated_length": 237.3,
|
|
"completions/mean_length": 187.4125,
|
|
"completions/mean_terminated_length": 139.9214309692383,
|
|
"completions/min_length": 81.0,
|
|
"completions/min_terminated_length": 81.0,
|
|
"entropy": 0.4397554527968168,
|
|
"epoch": 0.4775880469583778,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.33203125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.035601213574409485,
|
|
"num_tokens": 16776598.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.40189326405525205,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.40189328789711,
|
|
"step": 3580,
|
|
"step_time": 17.915511938370763,
|
|
"zero_adv_skip_rate": 0.3925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 760.4,
|
|
"completions/max_terminated_length": 462.4,
|
|
"completions/mean_length": 281.325,
|
|
"completions/mean_terminated_length": 236.0482162475586,
|
|
"completions/min_length": 105.8,
|
|
"completions/min_terminated_length": 105.8,
|
|
"entropy": 0.3660246117506176,
|
|
"epoch": 0.4789220917822839,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.00430905856192112,
|
|
"num_tokens": 16830080.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.15235702097415924,
|
|
"rewards/correctness_reward/mean": 0.2125,
|
|
"rewards/correctness_reward/std": 0.15235702097415924,
|
|
"step": 3590,
|
|
"step_time": 22.02548081725836,
|
|
"zero_adv_skip_rate": 0.393
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 708.5,
|
|
"completions/max_terminated_length": 461.6,
|
|
"completions/mean_length": 269.15,
|
|
"completions/mean_terminated_length": 227.3464340209961,
|
|
"completions/min_length": 84.4,
|
|
"completions/min_terminated_length": 84.4,
|
|
"entropy": 0.3618824081495404,
|
|
"epoch": 0.48025613660619,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.4609375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.17707740068435668,
|
|
"num_tokens": 16880052.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.2504029482603073,
|
|
"step": 3600,
|
|
"step_time": 20.953678432013838,
|
|
"zero_adv_skip_rate": 0.3931
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 421.5,
|
|
"completions/max_terminated_length": 421.5,
|
|
"completions/mean_length": 183.9125,
|
|
"completions/mean_terminated_length": 183.44107208251953,
|
|
"completions/min_length": 68.8,
|
|
"completions/min_terminated_length": 68.8,
|
|
"entropy": 0.3219907065853477,
|
|
"epoch": 0.4815901814300961,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5859375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.029902338981628418,
|
|
"num_tokens": 16924501.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.22051151394844054,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.2205115258693695,
|
|
"step": 3610,
|
|
"step_time": 12.944401746615767,
|
|
"zero_adv_skip_rate": 0.3934
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 437.6,
|
|
"completions/max_terminated_length": 435.0,
|
|
"completions/mean_length": 200.025,
|
|
"completions/mean_terminated_length": 199.28214340209962,
|
|
"completions/min_length": 73.2,
|
|
"completions/min_terminated_length": 73.2,
|
|
"entropy": 0.39117641846782397,
|
|
"epoch": 0.4829242262540021,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.3984375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0316151112318039,
|
|
"num_tokens": 16969359.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.3038551896810532,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.30385519564151764,
|
|
"step": 3620,
|
|
"step_time": 13.675478547811508,
|
|
"zero_adv_skip_rate": 0.3931
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 615.0,
|
|
"completions/max_terminated_length": 404.2,
|
|
"completions/mean_length": 204.175,
|
|
"completions/mean_terminated_length": 175.86428680419922,
|
|
"completions/min_length": 83.5,
|
|
"completions/min_terminated_length": 83.5,
|
|
"entropy": 0.3092529183874528,
|
|
"epoch": 0.4842582710779082,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.72265625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.14459167718887328,
|
|
"num_tokens": 17015493.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.2504029482603073,
|
|
"step": 3630,
|
|
"step_time": 18.3708520937711,
|
|
"zero_adv_skip_rate": 0.3931
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 315.6,
|
|
"completions/max_terminated_length": 310.7,
|
|
"completions/mean_length": 150.825,
|
|
"completions/mean_terminated_length": 149.95892868041992,
|
|
"completions/min_length": 57.1,
|
|
"completions/min_terminated_length": 57.1,
|
|
"entropy": 0.36279461719095707,
|
|
"epoch": 0.4855923159018143,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.08129200339317322,
|
|
"num_tokens": 17057895.0,
|
|
"reward": 0.5125,
|
|
"reward_std": 0.361081862449646,
|
|
"rewards/correctness_reward/mean": 0.5125,
|
|
"rewards/correctness_reward/std": 0.3610818684101105,
|
|
"step": 3640,
|
|
"step_time": 10.518262290861458,
|
|
"zero_adv_skip_rate": 0.3926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 899.7,
|
|
"completions/max_terminated_length": 540.4,
|
|
"completions/mean_length": 290.3,
|
|
"completions/mean_terminated_length": 243.03571624755858,
|
|
"completions/min_length": 115.5,
|
|
"completions/min_terminated_length": 115.5,
|
|
"entropy": 0.36562213940279825,
|
|
"epoch": 0.4869263607257204,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.30859375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.07637371420860291,
|
|
"num_tokens": 17110287.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.33287991881370543,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.33287993669509885,
|
|
"step": 3650,
|
|
"step_time": 26.465035409480333,
|
|
"zero_adv_skip_rate": 0.3923
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 440.3,
|
|
"completions/max_terminated_length": 293.8,
|
|
"completions/mean_length": 191.4,
|
|
"completions/mean_terminated_length": 170.00893020629883,
|
|
"completions/min_length": 87.7,
|
|
"completions/min_terminated_length": 87.7,
|
|
"entropy": 0.3425381085835397,
|
|
"epoch": 0.48826040554962646,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.07325968742370606,
|
|
"num_tokens": 17154151.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.23400336503982544,
|
|
"rewards/correctness_reward/mean": 0.5,
|
|
"rewards/correctness_reward/std": 0.23400336503982544,
|
|
"step": 3660,
|
|
"step_time": 13.656539129372685,
|
|
"zero_adv_skip_rate": 0.3923
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 557.0,
|
|
"completions/max_terminated_length": 435.5,
|
|
"completions/mean_length": 227.575,
|
|
"completions/mean_terminated_length": 202.52143096923828,
|
|
"completions/min_length": 91.8,
|
|
"completions/min_terminated_length": 91.8,
|
|
"entropy": 0.3681626107543707,
|
|
"epoch": 0.48959445037353255,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.4453125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.006588567793369293,
|
|
"num_tokens": 17202349.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.1375,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 3670,
|
|
"step_time": 16.89750128146261,
|
|
"zero_adv_skip_rate": 0.3926
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 363.8,
|
|
"completions/max_terminated_length": 356.2,
|
|
"completions/mean_length": 217.25,
|
|
"completions/mean_terminated_length": 216.0571434020996,
|
|
"completions/min_length": 106.8,
|
|
"completions/min_terminated_length": 106.8,
|
|
"entropy": 0.2736032985150814,
|
|
"epoch": 0.49092849519743864,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.029405826330184938,
|
|
"num_tokens": 17247017.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.24238284826278686,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.24238286018371583,
|
|
"step": 3680,
|
|
"step_time": 11.52172925202176,
|
|
"zero_adv_skip_rate": 0.3929
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 371.7,
|
|
"completions/max_terminated_length": 355.9,
|
|
"completions/mean_length": 208.8875,
|
|
"completions/mean_terminated_length": 205.84285888671874,
|
|
"completions/min_length": 106.4,
|
|
"completions/min_terminated_length": 106.4,
|
|
"entropy": 0.2968423863252004,
|
|
"epoch": 0.49226254002134473,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.84375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.01857403963804245,
|
|
"num_tokens": 17295048.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.13887301683425904,
|
|
"rewards/correctness_reward/mean": 0.175,
|
|
"rewards/correctness_reward/std": 0.13887301683425904,
|
|
"step": 3690,
|
|
"step_time": 11.44270655028522,
|
|
"zero_adv_skip_rate": 0.3938
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 688.6,
|
|
"completions/max_terminated_length": 639.6,
|
|
"completions/mean_length": 309.55,
|
|
"completions/mean_terminated_length": 293.2000030517578,
|
|
"completions/min_length": 116.8,
|
|
"completions/min_terminated_length": 116.8,
|
|
"entropy": 0.352532713940101,
|
|
"epoch": 0.4935965848452508,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.6875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0022005259990692137,
|
|
"num_tokens": 17351452.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.31648818850517274,
|
|
"rewards/correctness_reward/mean": 0.2,
|
|
"rewards/correctness_reward/std": 0.31648818850517274,
|
|
"step": 3700,
|
|
"step_time": 20.650383820943535,
|
|
"zero_adv_skip_rate": 0.3935
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 780.3,
|
|
"completions/max_terminated_length": 523.2,
|
|
"completions/mean_length": 283.325,
|
|
"completions/mean_terminated_length": 238.73750228881835,
|
|
"completions/min_length": 87.4,
|
|
"completions/min_terminated_length": 87.4,
|
|
"entropy": 0.49020378415783245,
|
|
"epoch": 0.4949306296691569,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.01145332157611847,
|
|
"num_tokens": 17404638.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.13340125977993011,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.1334012657403946,
|
|
"step": 3710,
|
|
"step_time": 22.614077482186257,
|
|
"zero_adv_skip_rate": 0.3943
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 1014.2,
|
|
"completions/max_terminated_length": 650.5,
|
|
"completions/mean_length": 321.825,
|
|
"completions/mean_terminated_length": 265.7142929077148,
|
|
"completions/min_length": 110.5,
|
|
"completions/min_terminated_length": 110.5,
|
|
"entropy": 0.41063491422683,
|
|
"epoch": 0.49626467449306294,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.267578125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.03764457106590271,
|
|
"num_tokens": 17458896.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.22598327100276946,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.22598327696323395,
|
|
"step": 3720,
|
|
"step_time": 29.43062931280583,
|
|
"zero_adv_skip_rate": 0.3946
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 893.2,
|
|
"completions/max_terminated_length": 587.5,
|
|
"completions/mean_length": 311.55,
|
|
"completions/mean_terminated_length": 266.2375015258789,
|
|
"completions/min_length": 113.1,
|
|
"completions/min_terminated_length": 113.1,
|
|
"entropy": 0.3127228822559118,
|
|
"epoch": 0.49759871931696903,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.32421875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.022151993215084077,
|
|
"num_tokens": 17512508.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.14603425860404967,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.14603425860404967,
|
|
"step": 3730,
|
|
"step_time": 25.851493115257473,
|
|
"zero_adv_skip_rate": 0.3954
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 874.0,
|
|
"completions/max_terminated_length": 594.7,
|
|
"completions/mean_length": 314.95,
|
|
"completions/mean_terminated_length": 228.38571586608887,
|
|
"completions/min_length": 75.2,
|
|
"completions/min_terminated_length": 75.2,
|
|
"entropy": 0.43628463389662403,
|
|
"epoch": 0.4989327641408751,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.6328125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.08842852711677551,
|
|
"num_tokens": 17567368.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.2777381867170334,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.27773819863796234,
|
|
"step": 3740,
|
|
"step_time": 25.75169428177178,
|
|
"zero_adv_skip_rate": 0.3955
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 741.7,
|
|
"completions/max_terminated_length": 382.7,
|
|
"completions/mean_length": 246.3875,
|
|
"completions/mean_terminated_length": 189.48750381469728,
|
|
"completions/min_length": 89.2,
|
|
"completions/min_terminated_length": 89.2,
|
|
"entropy": 0.35249204739617807,
|
|
"epoch": 0.5002668089647813,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.53515625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.006529676914215088,
|
|
"num_tokens": 17616231.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.2722664296627045,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.2722664475440979,
|
|
"step": 3750,
|
|
"step_time": 22.09202901730314,
|
|
"zero_adv_skip_rate": 0.3955
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 457.5,
|
|
"completions/max_terminated_length": 457.5,
|
|
"completions/mean_length": 190.8875,
|
|
"completions/mean_terminated_length": 190.8875,
|
|
"completions/min_length": 78.5,
|
|
"completions/min_terminated_length": 78.5,
|
|
"entropy": 0.44757911852664417,
|
|
"epoch": 0.5016008537886874,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.56640625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0663190245628357,
|
|
"num_tokens": 17665422.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.4090623527765274,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.4090623646974564,
|
|
"step": 3760,
|
|
"step_time": 14.723386551719159,
|
|
"zero_adv_skip_rate": 0.3947
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 810.6,
|
|
"completions/max_terminated_length": 566.7,
|
|
"completions/mean_length": 278.45,
|
|
"completions/mean_terminated_length": 233.28036193847657,
|
|
"completions/min_length": 97.1,
|
|
"completions/min_terminated_length": 97.1,
|
|
"entropy": 0.385783181215326,
|
|
"epoch": 0.5029348986125933,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.007521822303533554,
|
|
"num_tokens": 17717802.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.2558668524026871,
|
|
"rewards/correctness_reward/mean": 0.1875,
|
|
"rewards/correctness_reward/std": 0.25586686432361605,
|
|
"step": 3770,
|
|
"step_time": 24.078734265826643,
|
|
"zero_adv_skip_rate": 0.3947
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 733.4,
|
|
"completions/max_terminated_length": 400.3,
|
|
"completions/mean_length": 267.8875,
|
|
"completions/mean_terminated_length": 222.38928680419923,
|
|
"completions/min_length": 95.8,
|
|
"completions/min_terminated_length": 95.8,
|
|
"entropy": 0.37819914023081463,
|
|
"epoch": 0.5042689434364994,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.609375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.02871718108654022,
|
|
"num_tokens": 17765657.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.26849985122680664,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.2684998571872711,
|
|
"step": 3780,
|
|
"step_time": 22.100119402538986,
|
|
"zero_adv_skip_rate": 0.3947
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 420.7,
|
|
"completions/max_terminated_length": 420.7,
|
|
"completions/mean_length": 203.9875,
|
|
"completions/mean_terminated_length": 203.9875,
|
|
"completions/min_length": 90.9,
|
|
"completions/min_terminated_length": 90.9,
|
|
"entropy": 0.36337591824121773,
|
|
"epoch": 0.5056029882604055,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.451171875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0025206930935382844,
|
|
"num_tokens": 17816368.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.1632926881313324,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.1632926881313324,
|
|
"step": 3790,
|
|
"step_time": 13.254985020309686,
|
|
"zero_adv_skip_rate": 0.3953
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 654.1,
|
|
"completions/max_terminated_length": 471.0,
|
|
"completions/mean_length": 296.1125,
|
|
"completions/mean_terminated_length": 271.1553588867188,
|
|
"completions/min_length": 143.3,
|
|
"completions/min_terminated_length": 143.3,
|
|
"entropy": 0.2603771343516807,
|
|
"epoch": 0.5069370330843116,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.4453125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.11090404987335205,
|
|
"num_tokens": 17868121.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.28659676015377045,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.2865967661142349,
|
|
"step": 3800,
|
|
"step_time": 20.27777730477974,
|
|
"zero_adv_skip_rate": 0.3953
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 292.9,
|
|
"completions/max_terminated_length": 271.0,
|
|
"completions/mean_length": 141.25,
|
|
"completions/mean_terminated_length": 137.05000114440918,
|
|
"completions/min_length": 69.5,
|
|
"completions/min_terminated_length": 69.5,
|
|
"entropy": 0.3279303933493793,
|
|
"epoch": 0.5082710779082177,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.015526203811168671,
|
|
"num_tokens": 17908189.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 3810,
|
|
"step_time": 9.849577331729233,
|
|
"zero_adv_skip_rate": 0.3955
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 355.3,
|
|
"completions/max_terminated_length": 326.9,
|
|
"completions/mean_length": 176.0125,
|
|
"completions/mean_terminated_length": 170.96250076293944,
|
|
"completions/min_length": 74.3,
|
|
"completions/min_terminated_length": 74.3,
|
|
"entropy": 0.380780036428145,
|
|
"epoch": 0.5096051227321238,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04004402756690979,
|
|
"num_tokens": 17951806.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.3038551896810532,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.30385519564151764,
|
|
"step": 3820,
|
|
"step_time": 11.90960400197655,
|
|
"zero_adv_skip_rate": 0.3953
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 591.9,
|
|
"completions/max_terminated_length": 591.9,
|
|
"completions/mean_length": 223.425,
|
|
"completions/mean_terminated_length": 223.425,
|
|
"completions/min_length": 95.6,
|
|
"completions/min_terminated_length": 95.6,
|
|
"entropy": 0.33513430471066386,
|
|
"epoch": 0.5109391675560299,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.50390625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.022818847000598906,
|
|
"num_tokens": 18001608.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.3918039232492447,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.39180393517017365,
|
|
"step": 3830,
|
|
"step_time": 18.837568407319488,
|
|
"zero_adv_skip_rate": 0.3948
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 484.7,
|
|
"completions/max_terminated_length": 479.3,
|
|
"completions/mean_length": 175.875,
|
|
"completions/mean_terminated_length": 174.63214416503905,
|
|
"completions/min_length": 69.4,
|
|
"completions/min_terminated_length": 69.4,
|
|
"entropy": 0.37119448251490084,
|
|
"epoch": 0.512273212379936,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.267578125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.010186411440372467,
|
|
"num_tokens": 18041790.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.3147830098867416,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.31478302776813505,
|
|
"step": 3840,
|
|
"step_time": 15.435423675458878,
|
|
"zero_adv_skip_rate": 0.3945
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 320.2,
|
|
"completions/max_terminated_length": 312.6,
|
|
"completions/mean_length": 175.875,
|
|
"completions/mean_terminated_length": 173.05000076293945,
|
|
"completions/min_length": 81.6,
|
|
"completions/min_terminated_length": 81.6,
|
|
"entropy": 0.3975922116743667,
|
|
"epoch": 0.513607257203842,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04811665415763855,
|
|
"num_tokens": 18085508.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.32402919232845306,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.32402920424938203,
|
|
"step": 3850,
|
|
"step_time": 10.792845758236945,
|
|
"zero_adv_skip_rate": 0.3943
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 636.7,
|
|
"completions/max_terminated_length": 530.7,
|
|
"completions/mean_length": 250.7625,
|
|
"completions/mean_terminated_length": 228.3107162475586,
|
|
"completions/min_length": 99.2,
|
|
"completions/min_terminated_length": 99.2,
|
|
"entropy": 0.4205526700243354,
|
|
"epoch": 0.5149413020277481,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0038627587258815766,
|
|
"num_tokens": 18134385.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.15235702097415924,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.15235702097415924,
|
|
"step": 3860,
|
|
"step_time": 19.28283246587962,
|
|
"zero_adv_skip_rate": 0.3948
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 866.2,
|
|
"completions/max_terminated_length": 367.2,
|
|
"completions/mean_length": 257.7,
|
|
"completions/mean_terminated_length": 165.97440872192382,
|
|
"completions/min_length": 72.6,
|
|
"completions/min_terminated_length": 72.6,
|
|
"entropy": 0.30435697987143484,
|
|
"epoch": 0.5162753468516542,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.21958839893341064,
|
|
"num_tokens": 18182753.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.3475821644067764,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.3475821942090988,
|
|
"step": 3870,
|
|
"step_time": 26.35086447931826,
|
|
"zero_adv_skip_rate": 0.3946
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 850.1,
|
|
"completions/max_terminated_length": 349.9,
|
|
"completions/mean_length": 253.6125,
|
|
"completions/mean_terminated_length": 184.4250030517578,
|
|
"completions/min_length": 77.9,
|
|
"completions/min_terminated_length": 77.9,
|
|
"entropy": 0.3856626923329064,
|
|
"epoch": 0.5176093916755603,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.294921875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.07960601449012757,
|
|
"num_tokens": 18236570.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.3021578580141068,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.3021578699350357,
|
|
"step": 3880,
|
|
"step_time": 25.971985073015095,
|
|
"zero_adv_skip_rate": 0.3943
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 555.8,
|
|
"completions/max_terminated_length": 362.2,
|
|
"completions/mean_length": 217.125,
|
|
"completions/mean_terminated_length": 190.2303581237793,
|
|
"completions/min_length": 76.8,
|
|
"completions/min_terminated_length": 76.8,
|
|
"entropy": 0.40790089033544064,
|
|
"epoch": 0.5189434364994664,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.03186295330524445,
|
|
"num_tokens": 18282708.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.22220884561538695,
|
|
"rewards/correctness_reward/mean": 0.1625,
|
|
"rewards/correctness_reward/std": 0.22220885157585143,
|
|
"step": 3890,
|
|
"step_time": 17.18412474133074,
|
|
"zero_adv_skip_rate": 0.3946
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 537.8,
|
|
"completions/max_terminated_length": 415.6,
|
|
"completions/mean_length": 215.9625,
|
|
"completions/mean_terminated_length": 193.27143096923828,
|
|
"completions/min_length": 66.4,
|
|
"completions/min_terminated_length": 66.4,
|
|
"entropy": 0.29590357358877856,
|
|
"epoch": 0.5202774813233725,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.11023464202880859,
|
|
"num_tokens": 18339273.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.261338609457016,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.2613386154174805,
|
|
"step": 3900,
|
|
"step_time": 16.914482615049927,
|
|
"zero_adv_skip_rate": 0.3946
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 603.8,
|
|
"completions/max_terminated_length": 601.1,
|
|
"completions/mean_length": 243.4625,
|
|
"completions/mean_terminated_length": 242.70892868041992,
|
|
"completions/min_length": 88.9,
|
|
"completions/min_terminated_length": 88.9,
|
|
"entropy": 0.47644152771681547,
|
|
"epoch": 0.5216115261472786,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.02761513292789459,
|
|
"num_tokens": 18387526.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.2739637613296509,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.27396377325057986,
|
|
"step": 3910,
|
|
"step_time": 18.707430651504545,
|
|
"zero_adv_skip_rate": 0.3946
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 342.3,
|
|
"completions/max_terminated_length": 342.3,
|
|
"completions/mean_length": 125.5875,
|
|
"completions/mean_terminated_length": 125.5875,
|
|
"completions/min_length": 62.8,
|
|
"completions/min_terminated_length": 62.8,
|
|
"entropy": 0.39844806827604773,
|
|
"epoch": 0.5229455709711847,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.71875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.003868437185883522,
|
|
"num_tokens": 18425597.0,
|
|
"reward": 0.6875,
|
|
"reward_std": 0.22220884561538695,
|
|
"rewards/correctness_reward/mean": 0.6875,
|
|
"rewards/correctness_reward/std": 0.22220885157585143,
|
|
"step": 3920,
|
|
"step_time": 11.203698441013694,
|
|
"zero_adv_skip_rate": 0.3949
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 759.3,
|
|
"completions/max_terminated_length": 608.5,
|
|
"completions/mean_length": 371.675,
|
|
"completions/mean_terminated_length": 339.8660827636719,
|
|
"completions/min_length": 86.8,
|
|
"completions/min_terminated_length": 86.8,
|
|
"entropy": 0.36443599488120526,
|
|
"epoch": 0.5242796157950907,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.59765625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.05824517607688904,
|
|
"num_tokens": 18485587.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.3757762610912323,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.3757762908935547,
|
|
"step": 3930,
|
|
"step_time": 23.359687845781444,
|
|
"zero_adv_skip_rate": 0.3944
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 714.0,
|
|
"completions/max_terminated_length": 400.9,
|
|
"completions/mean_length": 253.725,
|
|
"completions/mean_terminated_length": 204.30178985595703,
|
|
"completions/min_length": 102.0,
|
|
"completions/min_terminated_length": 102.0,
|
|
"entropy": 0.28930187520260614,
|
|
"epoch": 0.5256136606189968,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.05356409549713135,
|
|
"num_tokens": 18536365.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.30129900574684143,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.30129902362823485,
|
|
"step": 3940,
|
|
"step_time": 21.845919458009302,
|
|
"zero_adv_skip_rate": 0.3944
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 620.8,
|
|
"completions/max_terminated_length": 458.6,
|
|
"completions/mean_length": 258.8125,
|
|
"completions/mean_terminated_length": 235.55178680419922,
|
|
"completions/min_length": 97.4,
|
|
"completions/min_terminated_length": 97.4,
|
|
"entropy": 0.320851506665349,
|
|
"epoch": 0.5269477054429029,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.337890625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.08559554815292358,
|
|
"num_tokens": 18586014.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.26133076250553133,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.26133078038692475,
|
|
"step": 3950,
|
|
"step_time": 19.23950833193958,
|
|
"zero_adv_skip_rate": 0.3944
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 238.9,
|
|
"completions/max_terminated_length": 237.2,
|
|
"completions/mean_length": 136.0125,
|
|
"completions/mean_terminated_length": 134.86250076293945,
|
|
"completions/min_length": 66.4,
|
|
"completions/min_terminated_length": 66.4,
|
|
"entropy": 0.32332486752420664,
|
|
"epoch": 0.528281750266809,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0253368079662323,
|
|
"num_tokens": 18624495.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.1595182627439499,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.1595182627439499,
|
|
"step": 3960,
|
|
"step_time": 8.067500183172523,
|
|
"zero_adv_skip_rate": 0.3949
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 380.3,
|
|
"completions/max_terminated_length": 379.7,
|
|
"completions/mean_length": 179.525,
|
|
"completions/mean_terminated_length": 179.32500038146972,
|
|
"completions/min_length": 78.6,
|
|
"completions/min_terminated_length": 78.6,
|
|
"entropy": 0.4730745820949475,
|
|
"epoch": 0.5296157950907151,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.71875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.009233638644218445,
|
|
"num_tokens": 18668481.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.2394672751426697,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.23946728110313414,
|
|
"step": 3970,
|
|
"step_time": 12.389599643554538,
|
|
"zero_adv_skip_rate": 0.395
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 625.1,
|
|
"completions/max_terminated_length": 544.3,
|
|
"completions/mean_length": 259.775,
|
|
"completions/mean_terminated_length": 240.08392944335938,
|
|
"completions/min_length": 102.8,
|
|
"completions/min_terminated_length": 102.8,
|
|
"entropy": 0.31730237377009224,
|
|
"epoch": 0.5309498399146211,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.014990584552288055,
|
|
"num_tokens": 18715983.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.3202547669410706,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.3202547788619995,
|
|
"step": 3980,
|
|
"step_time": 19.507628989126534,
|
|
"zero_adv_skip_rate": 0.3947
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 425.2,
|
|
"completions/max_terminated_length": 424.6,
|
|
"completions/mean_length": 186.8625,
|
|
"completions/mean_terminated_length": 186.59107208251953,
|
|
"completions/min_length": 83.2,
|
|
"completions/min_terminated_length": 83.2,
|
|
"entropy": 0.3084944443544373,
|
|
"epoch": 0.5322838847385272,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.408203125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1472116470336914,
|
|
"num_tokens": 18758308.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.36277919411659243,
|
|
"rewards/correctness_reward/mean": 0.5,
|
|
"rewards/correctness_reward/std": 0.36277919411659243,
|
|
"step": 3990,
|
|
"step_time": 14.007639886345714,
|
|
"zero_adv_skip_rate": 0.3942
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 482.1,
|
|
"completions/max_terminated_length": 354.2,
|
|
"completions/mean_length": 184.55,
|
|
"completions/mean_terminated_length": 162.35535888671876,
|
|
"completions/min_length": 66.6,
|
|
"completions/min_terminated_length": 66.6,
|
|
"entropy": 0.37358305882662535,
|
|
"epoch": 0.5336179295624333,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.00945272073149681,
|
|
"num_tokens": 18800288.0,
|
|
"reward": 0.525,
|
|
"reward_std": 0.20411193668842315,
|
|
"rewards/correctness_reward/mean": 0.525,
|
|
"rewards/correctness_reward/std": 0.20411194264888763,
|
|
"step": 4000,
|
|
"step_time": 15.152233835216611,
|
|
"zero_adv_skip_rate": 0.3945
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 808.0,
|
|
"completions/max_terminated_length": 611.2,
|
|
"completions/mean_length": 341.1125,
|
|
"completions/mean_terminated_length": 288.9011962890625,
|
|
"completions/min_length": 113.1,
|
|
"completions/min_terminated_length": 113.1,
|
|
"entropy": 0.3071100354815523,
|
|
"epoch": 0.5349519743863393,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0587286651134491,
|
|
"num_tokens": 18855409.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.14603425860404967,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.14603425860404967,
|
|
"step": 4010,
|
|
"step_time": 24.431909175403415,
|
|
"zero_adv_skip_rate": 0.3953
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 488.3,
|
|
"completions/max_terminated_length": 302.5,
|
|
"completions/mean_length": 183.9625,
|
|
"completions/mean_terminated_length": 158.53571472167968,
|
|
"completions/min_length": 61.9,
|
|
"completions/min_terminated_length": 61.9,
|
|
"entropy": 0.4408161534617345,
|
|
"epoch": 0.5362860192102454,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.001962876319885254,
|
|
"num_tokens": 18901278.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.14603425860404967,
|
|
"rewards/correctness_reward/mean": 0.35,
|
|
"rewards/correctness_reward/std": 0.14603425860404967,
|
|
"step": 4020,
|
|
"step_time": 15.11487988755107,
|
|
"zero_adv_skip_rate": 0.396
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 486.7,
|
|
"completions/max_terminated_length": 483.5,
|
|
"completions/mean_length": 229.6,
|
|
"completions/mean_terminated_length": 229.0035717010498,
|
|
"completions/min_length": 99.0,
|
|
"completions/min_terminated_length": 99.0,
|
|
"entropy": 0.30205688268567127,
|
|
"epoch": 0.5376200640341515,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.92578125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.07290530800819398,
|
|
"num_tokens": 18947542.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.2449311852455139,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.24493119716644288,
|
|
"step": 4030,
|
|
"step_time": 15.47567258523777,
|
|
"zero_adv_skip_rate": 0.396
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 741.7,
|
|
"completions/max_terminated_length": 428.8,
|
|
"completions/mean_length": 281.8125,
|
|
"completions/mean_terminated_length": 208.50714492797852,
|
|
"completions/min_length": 95.6,
|
|
"completions/min_terminated_length": 95.6,
|
|
"entropy": 0.28488827776163816,
|
|
"epoch": 0.5389541088580576,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.390625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.009341034293174743,
|
|
"num_tokens": 19002335.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.08711025416851044,
|
|
"rewards/correctness_reward/mean": 0.15,
|
|
"rewards/correctness_reward/std": 0.08711026012897491,
|
|
"step": 4040,
|
|
"step_time": 22.255815756507218,
|
|
"zero_adv_skip_rate": 0.397
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 604.1,
|
|
"completions/max_terminated_length": 411.2,
|
|
"completions/mean_length": 217.7375,
|
|
"completions/mean_terminated_length": 189.8321445465088,
|
|
"completions/min_length": 70.1,
|
|
"completions/min_terminated_length": 70.1,
|
|
"entropy": 0.5010717622935772,
|
|
"epoch": 0.5402881536819637,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.02596483826637268,
|
|
"num_tokens": 19056490.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.11700168251991272,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.11700168251991272,
|
|
"step": 4050,
|
|
"step_time": 18.450345692876727,
|
|
"zero_adv_skip_rate": 0.3978
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0875,
|
|
"completions/max_length": 881.1,
|
|
"completions/max_terminated_length": 421.9,
|
|
"completions/mean_length": 322.125,
|
|
"completions/mean_terminated_length": 233.33572006225586,
|
|
"completions/min_length": 94.5,
|
|
"completions/min_terminated_length": 94.5,
|
|
"entropy": 0.33020643470808864,
|
|
"epoch": 0.5416221985058698,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.023578400909900665,
|
|
"num_tokens": 19113548.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.38463483452796937,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.38463485836982725,
|
|
"step": 4060,
|
|
"step_time": 27.13537761569023,
|
|
"zero_adv_skip_rate": 0.3973
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 516.3,
|
|
"completions/max_terminated_length": 515.9,
|
|
"completions/mean_length": 230.6,
|
|
"completions/mean_terminated_length": 230.3035717010498,
|
|
"completions/min_length": 84.8,
|
|
"completions/min_terminated_length": 84.8,
|
|
"entropy": 0.30481575690209867,
|
|
"epoch": 0.5429562433297759,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.67578125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.025434333086013793,
|
|
"num_tokens": 19163468.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.2112731784582138,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.2112731844186783,
|
|
"step": 4070,
|
|
"step_time": 16.33164927996695,
|
|
"zero_adv_skip_rate": 0.3975
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 804.5,
|
|
"completions/max_terminated_length": 532.9,
|
|
"completions/mean_length": 309.6125,
|
|
"completions/mean_terminated_length": 266.682145690918,
|
|
"completions/min_length": 113.2,
|
|
"completions/min_terminated_length": 113.2,
|
|
"entropy": 0.3610837352462113,
|
|
"epoch": 0.544290288153682,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04649161696434021,
|
|
"num_tokens": 19217861.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.17422050833702088,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.17422052025794982,
|
|
"step": 4080,
|
|
"step_time": 24.34745808392763,
|
|
"zero_adv_skip_rate": 0.398
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 463.3,
|
|
"completions/max_terminated_length": 352.2,
|
|
"completions/mean_length": 209.5625,
|
|
"completions/mean_terminated_length": 190.21428833007812,
|
|
"completions/min_length": 90.6,
|
|
"completions/min_terminated_length": 90.6,
|
|
"entropy": 0.31294114189222455,
|
|
"epoch": 0.5456243329775881,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.60546875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.01792086511850357,
|
|
"num_tokens": 19261818.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.1414213538169861,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.1414213538169861,
|
|
"step": 4090,
|
|
"step_time": 14.765348818711937,
|
|
"zero_adv_skip_rate": 0.3985
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 814.1,
|
|
"completions/max_terminated_length": 481.8,
|
|
"completions/mean_length": 286.6625,
|
|
"completions/mean_terminated_length": 239.91428604125977,
|
|
"completions/min_length": 89.1,
|
|
"completions/min_terminated_length": 89.1,
|
|
"entropy": 0.38023844530107453,
|
|
"epoch": 0.5469583778014941,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.734375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1776528000831604,
|
|
"num_tokens": 19314287.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.3337466180324554,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.3337466180324554,
|
|
"step": 4100,
|
|
"step_time": 25.3570060598664,
|
|
"zero_adv_skip_rate": 0.398
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 377.5,
|
|
"completions/max_terminated_length": 377.5,
|
|
"completions/mean_length": 183.8125,
|
|
"completions/mean_terminated_length": 183.8125,
|
|
"completions/min_length": 79.9,
|
|
"completions/min_terminated_length": 79.9,
|
|
"entropy": 0.3154337767822047,
|
|
"epoch": 0.5482924226254002,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.6875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04845768213272095,
|
|
"num_tokens": 19356968.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.2575641840696335,
|
|
"rewards/correctness_reward/mean": 0.5,
|
|
"rewards/correctness_reward/std": 0.25756419003009795,
|
|
"step": 4110,
|
|
"step_time": 12.690755020920188,
|
|
"zero_adv_skip_rate": 0.3981
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 862.5,
|
|
"completions/max_terminated_length": 546.4,
|
|
"completions/mean_length": 246.425,
|
|
"completions/mean_terminated_length": 198.6125030517578,
|
|
"completions/min_length": 78.4,
|
|
"completions/min_terminated_length": 78.4,
|
|
"entropy": 0.3906542385617892,
|
|
"epoch": 0.5496264674493063,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.69140625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.10482479333877563,
|
|
"num_tokens": 19406306.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.28489942848682404,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.284899440407753,
|
|
"step": 4120,
|
|
"step_time": 26.72215065555647,
|
|
"zero_adv_skip_rate": 0.3981
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 642.2,
|
|
"completions/max_terminated_length": 320.4,
|
|
"completions/mean_length": 236.525,
|
|
"completions/mean_terminated_length": 190.23214569091797,
|
|
"completions/min_length": 93.5,
|
|
"completions/min_terminated_length": 93.5,
|
|
"entropy": 0.31346632577478883,
|
|
"epoch": 0.5509605122732124,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.578125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.004867070913314819,
|
|
"num_tokens": 19458052.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.19864802658557892,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.19864802658557892,
|
|
"step": 4130,
|
|
"step_time": 20.13130561225116,
|
|
"zero_adv_skip_rate": 0.3983
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 414.3,
|
|
"completions/max_terminated_length": 414.3,
|
|
"completions/mean_length": 198.8875,
|
|
"completions/mean_terminated_length": 198.8875,
|
|
"completions/min_length": 91.1,
|
|
"completions/min_terminated_length": 91.1,
|
|
"entropy": 0.4538300980089439,
|
|
"epoch": 0.5522945570971185,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.8671875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.08193295001983643,
|
|
"num_tokens": 19506595.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.36909410953521726,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.36909412145614623,
|
|
"step": 4140,
|
|
"step_time": 14.294465378951282,
|
|
"zero_adv_skip_rate": 0.3976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 1067.7,
|
|
"completions/max_terminated_length": 458.8,
|
|
"completions/mean_length": 296.3375,
|
|
"completions/mean_terminated_length": 203.64643325805665,
|
|
"completions/min_length": 89.3,
|
|
"completions/min_terminated_length": 89.3,
|
|
"entropy": 0.3533428443595767,
|
|
"epoch": 0.5536286019210246,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.009001788496971131,
|
|
"num_tokens": 19558806.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.23144718110561371,
|
|
"rewards/correctness_reward/mean": 0.2375,
|
|
"rewards/correctness_reward/std": 0.23144719302654265,
|
|
"step": 4150,
|
|
"step_time": 32.794452272262426,
|
|
"zero_adv_skip_rate": 0.3978
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 607.4,
|
|
"completions/max_terminated_length": 428.7,
|
|
"completions/mean_length": 219.8625,
|
|
"completions/mean_terminated_length": 196.4607147216797,
|
|
"completions/min_length": 80.9,
|
|
"completions/min_terminated_length": 80.9,
|
|
"entropy": 0.40464938394725325,
|
|
"epoch": 0.5549626467449307,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.5625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.013208796083927155,
|
|
"num_tokens": 19603867.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.21504760384559632,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.2150476098060608,
|
|
"step": 4160,
|
|
"step_time": 19.267009518295527,
|
|
"zero_adv_skip_rate": 0.3981
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 629.5,
|
|
"completions/max_terminated_length": 325.2,
|
|
"completions/mean_length": 262.8,
|
|
"completions/mean_terminated_length": 199.0970260620117,
|
|
"completions/min_length": 90.2,
|
|
"completions/min_terminated_length": 90.2,
|
|
"entropy": 0.3482857032989462,
|
|
"epoch": 0.5562966915688368,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.007823927700519562,
|
|
"num_tokens": 19652643.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.28829409182071686,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.28829409182071686,
|
|
"step": 4170,
|
|
"step_time": 20.2280963989906,
|
|
"zero_adv_skip_rate": 0.3981
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 822.3,
|
|
"completions/max_terminated_length": 822.3,
|
|
"completions/mean_length": 305.2125,
|
|
"completions/mean_terminated_length": 305.2125,
|
|
"completions/min_length": 88.2,
|
|
"completions/min_terminated_length": 88.2,
|
|
"entropy": 0.39519624629368383,
|
|
"epoch": 0.5576307363927427,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.011613527685403824,
|
|
"num_tokens": 19707820.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.27397160828113554,
|
|
"rewards/correctness_reward/mean": 0.2125,
|
|
"rewards/correctness_reward/std": 0.27397160828113554,
|
|
"step": 4180,
|
|
"step_time": 25.722647099662574,
|
|
"zero_adv_skip_rate": 0.3981
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 557.7,
|
|
"completions/max_terminated_length": 485.8,
|
|
"completions/mean_length": 258.225,
|
|
"completions/mean_terminated_length": 243.27500381469727,
|
|
"completions/min_length": 114.3,
|
|
"completions/min_terminated_length": 114.3,
|
|
"entropy": 0.3907346136868,
|
|
"epoch": 0.5589647812166488,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 1.328125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04020420908927917,
|
|
"num_tokens": 19763126.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.2276727557182312,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.22767276763916017,
|
|
"step": 4190,
|
|
"step_time": 17.909372388850898,
|
|
"zero_adv_skip_rate": 0.3983
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 833.6,
|
|
"completions/max_terminated_length": 706.0,
|
|
"completions/mean_length": 315.0,
|
|
"completions/mean_terminated_length": 294.84286041259764,
|
|
"completions/min_length": 100.9,
|
|
"completions/min_terminated_length": 100.9,
|
|
"entropy": 0.4069986803723233,
|
|
"epoch": 0.5602988260405549,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.84375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.027409684658050538,
|
|
"num_tokens": 19818166.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.2765199452638626,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.2765199452638626,
|
|
"step": 4200,
|
|
"step_time": 26.260944322217256,
|
|
"zero_adv_skip_rate": 0.3981
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 325.8,
|
|
"completions/max_terminated_length": 325.8,
|
|
"completions/mean_length": 168.025,
|
|
"completions/mean_terminated_length": 168.025,
|
|
"completions/min_length": 88.9,
|
|
"completions/min_terminated_length": 88.9,
|
|
"entropy": 0.29775026836432517,
|
|
"epoch": 0.561632870864461,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.6875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.10443680286407471,
|
|
"num_tokens": 19860448.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.35561010539531707,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.35561011731624603,
|
|
"step": 4210,
|
|
"step_time": 11.491556483600288,
|
|
"zero_adv_skip_rate": 0.3976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 261.6,
|
|
"completions/max_terminated_length": 257.9,
|
|
"completions/mean_length": 143.5625,
|
|
"completions/mean_terminated_length": 141.4321434020996,
|
|
"completions/min_length": 66.5,
|
|
"completions/min_terminated_length": 66.5,
|
|
"entropy": 0.38357098480420454,
|
|
"epoch": 0.5629669156883671,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04571352005004883,
|
|
"num_tokens": 19899029.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.3475821644067764,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.3475821942090988,
|
|
"step": 4220,
|
|
"step_time": 9.479561340436339,
|
|
"zero_adv_skip_rate": 0.3974
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 642.4,
|
|
"completions/max_terminated_length": 566.3,
|
|
"completions/mean_length": 331.4625,
|
|
"completions/mean_terminated_length": 312.9357192993164,
|
|
"completions/min_length": 111.0,
|
|
"completions/min_terminated_length": 111.0,
|
|
"entropy": 0.28833848354406655,
|
|
"epoch": 0.5643009605122732,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.5390625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.057658451795578006,
|
|
"num_tokens": 19958610.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.18515617549419403,
|
|
"rewards/correctness_reward/mean": 0.1875,
|
|
"rewards/correctness_reward/std": 0.185156187415123,
|
|
"step": 4230,
|
|
"step_time": 20.212628595158456,
|
|
"zero_adv_skip_rate": 0.3979
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 482.3,
|
|
"completions/max_terminated_length": 307.5,
|
|
"completions/mean_length": 177.375,
|
|
"completions/mean_terminated_length": 153.31785926818847,
|
|
"completions/min_length": 78.7,
|
|
"completions/min_terminated_length": 78.7,
|
|
"entropy": 0.3318137222280105,
|
|
"epoch": 0.5656350053361793,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.65234375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.015520358085632324,
|
|
"num_tokens": 20003208.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.43262317180633547,
|
|
"rewards/correctness_reward/mean": 0.3375,
|
|
"rewards/correctness_reward/std": 0.4326231896877289,
|
|
"step": 4240,
|
|
"step_time": 16.407854665443303,
|
|
"zero_adv_skip_rate": 0.3972
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 504.2,
|
|
"completions/max_terminated_length": 322.2,
|
|
"completions/mean_length": 217.3125,
|
|
"completions/mean_terminated_length": 188.58393440246581,
|
|
"completions/min_length": 111.9,
|
|
"completions/min_terminated_length": 111.9,
|
|
"entropy": 0.3514885837212205,
|
|
"epoch": 0.5669690501600854,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.71484375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.044480806589126586,
|
|
"num_tokens": 20047969.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.26680251955986023,
|
|
"rewards/correctness_reward/mean": 0.45,
|
|
"rewards/correctness_reward/std": 0.2668025314807892,
|
|
"step": 4250,
|
|
"step_time": 16.752388597559182,
|
|
"zero_adv_skip_rate": 0.3972
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.6,
|
|
"completions/max_terminated_length": 314.6,
|
|
"completions/mean_length": 163.6875,
|
|
"completions/mean_terminated_length": 163.6875,
|
|
"completions/min_length": 78.3,
|
|
"completions/min_terminated_length": 78.3,
|
|
"entropy": 0.3479438595887687,
|
|
"epoch": 0.5683030949839915,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.5546875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.007739922404289246,
|
|
"num_tokens": 20089832.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.38550153374671936,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.3855015397071838,
|
|
"step": 4260,
|
|
"step_time": 11.28819179693237,
|
|
"zero_adv_skip_rate": 0.3965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 447.0,
|
|
"completions/max_terminated_length": 308.2,
|
|
"completions/mean_length": 199.7125,
|
|
"completions/mean_terminated_length": 177.5678596496582,
|
|
"completions/min_length": 82.5,
|
|
"completions/min_terminated_length": 82.5,
|
|
"entropy": 0.36325254337862134,
|
|
"epoch": 0.5696371398078975,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 1.0703125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.002238249033689499,
|
|
"num_tokens": 20135761.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.3501461952924728,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.35014620125293733,
|
|
"step": 4270,
|
|
"step_time": 15.225613095797598,
|
|
"zero_adv_skip_rate": 0.396
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 553.2,
|
|
"completions/max_terminated_length": 544.7,
|
|
"completions/mean_length": 273.35,
|
|
"completions/mean_terminated_length": 271.39464416503904,
|
|
"completions/min_length": 119.7,
|
|
"completions/min_terminated_length": 119.7,
|
|
"entropy": 0.32961730206651346,
|
|
"epoch": 0.5709711846318036,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.050885021686553955,
|
|
"num_tokens": 20185981.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.34211825430393217,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.3421182781457901,
|
|
"step": 4280,
|
|
"step_time": 18.271826083771884,
|
|
"zero_adv_skip_rate": 0.3958
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 703.9,
|
|
"completions/max_terminated_length": 501.9,
|
|
"completions/mean_length": 301.2375,
|
|
"completions/mean_terminated_length": 267.6321479797363,
|
|
"completions/min_length": 108.5,
|
|
"completions/min_terminated_length": 108.5,
|
|
"entropy": 0.42995889857411385,
|
|
"epoch": 0.5723052294557097,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.07434165477752686,
|
|
"num_tokens": 20242784.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.17045392990112304,
|
|
"rewards/correctness_reward/mean": 0.225,
|
|
"rewards/correctness_reward/std": 0.17045392990112304,
|
|
"step": 4290,
|
|
"step_time": 22.216766913421452,
|
|
"zero_adv_skip_rate": 0.3963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 692.1,
|
|
"completions/max_terminated_length": 319.9,
|
|
"completions/mean_length": 193.225,
|
|
"completions/mean_terminated_length": 144.97857284545898,
|
|
"completions/min_length": 64.6,
|
|
"completions/min_terminated_length": 64.6,
|
|
"entropy": 0.3296644654391067,
|
|
"epoch": 0.5736392742796158,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 1.0859375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.03919318914413452,
|
|
"num_tokens": 20285090.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.31479085683822633,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.3147908627986908,
|
|
"step": 4300,
|
|
"step_time": 22.50000362414867,
|
|
"zero_adv_skip_rate": 0.396
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 639.9,
|
|
"completions/max_terminated_length": 382.7,
|
|
"completions/mean_length": 235.775,
|
|
"completions/mean_terminated_length": 189.43393249511718,
|
|
"completions/min_length": 89.4,
|
|
"completions/min_terminated_length": 89.4,
|
|
"entropy": 0.3697121351957321,
|
|
"epoch": 0.5749733191035219,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04056672155857086,
|
|
"num_tokens": 20339080.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.19864802658557892,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.19864802658557892,
|
|
"step": 4310,
|
|
"step_time": 20.519821687508376,
|
|
"zero_adv_skip_rate": 0.3963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 473.1,
|
|
"completions/max_terminated_length": 470.9,
|
|
"completions/mean_length": 204.5625,
|
|
"completions/mean_terminated_length": 203.1857147216797,
|
|
"completions/min_length": 72.4,
|
|
"completions/min_terminated_length": 72.4,
|
|
"entropy": 0.43942717683967203,
|
|
"epoch": 0.576307363927428,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.451171875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.05834516882896423,
|
|
"num_tokens": 20384317.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.3484488636255264,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.34844887554645537,
|
|
"step": 4320,
|
|
"step_time": 16.111680057272316,
|
|
"zero_adv_skip_rate": 0.3958
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 401.1,
|
|
"completions/max_terminated_length": 400.9,
|
|
"completions/mean_length": 172.575,
|
|
"completions/mean_terminated_length": 172.25535774230957,
|
|
"completions/min_length": 63.8,
|
|
"completions/min_terminated_length": 63.8,
|
|
"entropy": 0.41098425971964997,
|
|
"epoch": 0.5776414087513341,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.03774624764919281,
|
|
"num_tokens": 20427075.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.23400336503982544,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.23400336503982544,
|
|
"step": 4330,
|
|
"step_time": 13.857920923363418,
|
|
"zero_adv_skip_rate": 0.3958
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 563.3,
|
|
"completions/max_terminated_length": 560.2,
|
|
"completions/mean_length": 222.55,
|
|
"completions/mean_terminated_length": 221.15357208251953,
|
|
"completions/min_length": 69.4,
|
|
"completions/min_terminated_length": 69.4,
|
|
"entropy": 0.38097010254859925,
|
|
"epoch": 0.5789754535752402,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.216796875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0180676132440567,
|
|
"num_tokens": 20471375.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.23691893815994264,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.2369189441204071,
|
|
"step": 4340,
|
|
"step_time": 18.491527982335537,
|
|
"zero_adv_skip_rate": 0.3961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 830.2,
|
|
"completions/max_terminated_length": 514.0,
|
|
"completions/mean_length": 298.4625,
|
|
"completions/mean_terminated_length": 253.28214645385742,
|
|
"completions/min_length": 99.7,
|
|
"completions/min_terminated_length": 99.7,
|
|
"entropy": 0.3842758624814451,
|
|
"epoch": 0.5803094983991463,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.267578125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.005583821609616279,
|
|
"num_tokens": 20525588.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.23400336503982544,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.23400336503982544,
|
|
"step": 4350,
|
|
"step_time": 26.611478140298278,
|
|
"zero_adv_skip_rate": 0.3961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 545.6,
|
|
"completions/max_terminated_length": 540.3,
|
|
"completions/mean_length": 259.675,
|
|
"completions/mean_terminated_length": 258.1142860412598,
|
|
"completions/min_length": 85.8,
|
|
"completions/min_terminated_length": 85.8,
|
|
"entropy": 0.3309289346049939,
|
|
"epoch": 0.5816435432230523,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.02529134154319763,
|
|
"num_tokens": 20576362.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.3147830098867416,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.31478302776813505,
|
|
"step": 4360,
|
|
"step_time": 18.198992130346596,
|
|
"zero_adv_skip_rate": 0.3959
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 507.6,
|
|
"completions/max_terminated_length": 399.0,
|
|
"completions/mean_length": 208.1625,
|
|
"completions/mean_terminated_length": 180.45179138183593,
|
|
"completions/min_length": 89.8,
|
|
"completions/min_terminated_length": 89.8,
|
|
"entropy": 0.3836222554423979,
|
|
"epoch": 0.5829775880469584,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.015581783652305604,
|
|
"num_tokens": 20624231.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.321952098608017,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.32195210456848145,
|
|
"step": 4370,
|
|
"step_time": 17.05520320897922,
|
|
"zero_adv_skip_rate": 0.3957
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 775.6,
|
|
"completions/max_terminated_length": 669.4,
|
|
"completions/mean_length": 286.75,
|
|
"completions/mean_terminated_length": 267.59107666015626,
|
|
"completions/min_length": 98.5,
|
|
"completions/min_terminated_length": 98.5,
|
|
"entropy": 0.3040894381701946,
|
|
"epoch": 0.5843116328708644,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.275390625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.16031795740127563,
|
|
"num_tokens": 20676067.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.2394672751426697,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.23946728110313414,
|
|
"step": 4380,
|
|
"step_time": 24.850786171946673,
|
|
"zero_adv_skip_rate": 0.3957
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 595.6,
|
|
"completions/max_terminated_length": 595.6,
|
|
"completions/mean_length": 253.475,
|
|
"completions/mean_terminated_length": 253.475,
|
|
"completions/min_length": 118.0,
|
|
"completions/min_terminated_length": 118.0,
|
|
"entropy": 0.3543485472910106,
|
|
"epoch": 0.5856456776947705,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.69921875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.08229753971099854,
|
|
"num_tokens": 20723801.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.3648484408855438,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.3648484587669373,
|
|
"step": 4390,
|
|
"step_time": 19.801282063126564,
|
|
"zero_adv_skip_rate": 0.3952
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 305.7,
|
|
"completions/max_terminated_length": 302.9,
|
|
"completions/mean_length": 172.4375,
|
|
"completions/mean_terminated_length": 169.76964416503907,
|
|
"completions/min_length": 104.2,
|
|
"completions/min_terminated_length": 104.2,
|
|
"entropy": 0.31101572917153436,
|
|
"epoch": 0.5869797225186766,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.030564534664154052,
|
|
"num_tokens": 20764868.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.24662851691246032,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.2466285228729248,
|
|
"step": 4400,
|
|
"step_time": 11.75364425946027,
|
|
"zero_adv_skip_rate": 0.3952
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 628.5,
|
|
"completions/max_terminated_length": 605.3,
|
|
"completions/mean_length": 253.35,
|
|
"completions/mean_terminated_length": 230.93928756713868,
|
|
"completions/min_length": 90.6,
|
|
"completions/min_terminated_length": 90.6,
|
|
"entropy": 0.2928723325021565,
|
|
"epoch": 0.5883137673425827,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04644075632095337,
|
|
"num_tokens": 20815848.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.0816463440656662,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.0816463440656662,
|
|
"step": 4410,
|
|
"step_time": 19.7824366918765,
|
|
"zero_adv_skip_rate": 0.3961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 425.8,
|
|
"completions/max_terminated_length": 425.8,
|
|
"completions/mean_length": 196.675,
|
|
"completions/mean_terminated_length": 196.675,
|
|
"completions/min_length": 100.9,
|
|
"completions/min_terminated_length": 100.9,
|
|
"entropy": 0.3432732160602297,
|
|
"epoch": 0.5896478121664888,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.06701961755752564,
|
|
"num_tokens": 20859758.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.32402134537696836,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.3240213692188263,
|
|
"step": 4420,
|
|
"step_time": 14.589964005351067,
|
|
"zero_adv_skip_rate": 0.3959
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 652.9,
|
|
"completions/max_terminated_length": 347.9,
|
|
"completions/mean_length": 246.425,
|
|
"completions/mean_terminated_length": 184.4630989074707,
|
|
"completions/min_length": 82.0,
|
|
"completions/min_terminated_length": 82.0,
|
|
"entropy": 0.3370536897952358,
|
|
"epoch": 0.5909818569903948,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.06275556683540344,
|
|
"num_tokens": 20907648.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.2575641840696335,
|
|
"rewards/correctness_reward/mean": 0.3,
|
|
"rewards/correctness_reward/std": 0.25756419003009795,
|
|
"step": 4430,
|
|
"step_time": 20.998921237234025,
|
|
"zero_adv_skip_rate": 0.3959
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 819.8,
|
|
"completions/max_terminated_length": 373.4,
|
|
"completions/mean_length": 278.0375,
|
|
"completions/mean_terminated_length": 192.8761962890625,
|
|
"completions/min_length": 99.5,
|
|
"completions/min_terminated_length": 99.5,
|
|
"entropy": 0.32548070991677897,
|
|
"epoch": 0.5923159018143009,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.12487641572952271,
|
|
"num_tokens": 20958123.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.3438155859708786,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.34381560385227206,
|
|
"step": 4440,
|
|
"step_time": 26.02535605067387,
|
|
"zero_adv_skip_rate": 0.3957
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 865.2,
|
|
"completions/max_terminated_length": 713.6,
|
|
"completions/mean_length": 323.725,
|
|
"completions/mean_terminated_length": 299.0375015258789,
|
|
"completions/min_length": 106.2,
|
|
"completions/min_terminated_length": 106.2,
|
|
"entropy": 0.5627278657630086,
|
|
"epoch": 0.593649946638207,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.001958058774471283,
|
|
"num_tokens": 21017349.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.08711025416851044,
|
|
"rewards/correctness_reward/mean": 0.15,
|
|
"rewards/correctness_reward/std": 0.08711026012897491,
|
|
"step": 4450,
|
|
"step_time": 27.088518462795765,
|
|
"zero_adv_skip_rate": 0.3966
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 835.9,
|
|
"completions/max_terminated_length": 442.8,
|
|
"completions/mean_length": 241.575,
|
|
"completions/mean_terminated_length": 188.84285888671874,
|
|
"completions/min_length": 83.6,
|
|
"completions/min_terminated_length": 83.6,
|
|
"entropy": 0.3968210903306802,
|
|
"epoch": 0.5949839914621131,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.71875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.09398115873336792,
|
|
"num_tokens": 21064243.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.2777381867170334,
|
|
"rewards/correctness_reward/mean": 0.2125,
|
|
"rewards/correctness_reward/std": 0.27773819863796234,
|
|
"step": 4460,
|
|
"step_time": 27.17564995996654,
|
|
"zero_adv_skip_rate": 0.3966
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 400.7,
|
|
"completions/max_terminated_length": 399.2,
|
|
"completions/mean_length": 214.5625,
|
|
"completions/mean_terminated_length": 214.18035736083985,
|
|
"completions/min_length": 86.2,
|
|
"completions/min_terminated_length": 86.2,
|
|
"entropy": 0.3931025465329488,
|
|
"epoch": 0.5963180362860192,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0022782981395721436,
|
|
"num_tokens": 21110568.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.2811328500509262,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.2811328500509262,
|
|
"step": 4470,
|
|
"step_time": 13.57993810819462,
|
|
"zero_adv_skip_rate": 0.3966
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 495.4,
|
|
"completions/max_terminated_length": 484.4,
|
|
"completions/mean_length": 211.6,
|
|
"completions/mean_terminated_length": 209.01964416503907,
|
|
"completions/min_length": 80.6,
|
|
"completions/min_terminated_length": 80.6,
|
|
"entropy": 0.31391415496667224,
|
|
"epoch": 0.5976520811099253,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.353515625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.010071545094251632,
|
|
"num_tokens": 21155648.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.2558668524026871,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.25586686432361605,
|
|
"step": 4480,
|
|
"step_time": 16.77685091169551,
|
|
"zero_adv_skip_rate": 0.3967
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 524.2,
|
|
"completions/max_terminated_length": 351.5,
|
|
"completions/mean_length": 207.25,
|
|
"completions/mean_terminated_length": 182.95535888671876,
|
|
"completions/min_length": 76.5,
|
|
"completions/min_terminated_length": 76.5,
|
|
"entropy": 0.4006786613414685,
|
|
"epoch": 0.5989861259338314,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.39453125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04084830582141876,
|
|
"num_tokens": 21201252.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.261338609457016,
|
|
"rewards/correctness_reward/mean": 0.2125,
|
|
"rewards/correctness_reward/std": 0.2613386154174805,
|
|
"step": 4490,
|
|
"step_time": 17.77264038603753,
|
|
"zero_adv_skip_rate": 0.3967
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 671.8,
|
|
"completions/max_terminated_length": 477.5,
|
|
"completions/mean_length": 254.625,
|
|
"completions/mean_terminated_length": 229.61785812377929,
|
|
"completions/min_length": 103.5,
|
|
"completions/min_terminated_length": 103.5,
|
|
"entropy": 0.42134177188078564,
|
|
"epoch": 0.6003201707577375,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.06465117931365967,
|
|
"num_tokens": 21251294.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.13340125977993011,
|
|
"rewards/correctness_reward/mean": 0.325,
|
|
"rewards/correctness_reward/std": 0.1334012657403946,
|
|
"step": 4500,
|
|
"step_time": 21.741853268910198,
|
|
"zero_adv_skip_rate": 0.3973
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 744.4,
|
|
"completions/max_terminated_length": 603.0,
|
|
"completions/mean_length": 313.05,
|
|
"completions/mean_terminated_length": 291.7089305877686,
|
|
"completions/min_length": 103.7,
|
|
"completions/min_terminated_length": 103.7,
|
|
"entropy": 0.34113963196674985,
|
|
"epoch": 0.6016542155816436,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0906272292137146,
|
|
"num_tokens": 21304034.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.15696207880973817,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.1569620907306671,
|
|
"step": 4510,
|
|
"step_time": 24.17186191789806,
|
|
"zero_adv_skip_rate": 0.398
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 495.4,
|
|
"completions/max_terminated_length": 348.2,
|
|
"completions/mean_length": 179.8125,
|
|
"completions/mean_terminated_length": 156.9839309692383,
|
|
"completions/min_length": 84.9,
|
|
"completions/min_terminated_length": 84.9,
|
|
"entropy": 0.3159849625080824,
|
|
"epoch": 0.6029882604055496,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.310546875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.07340635061264038,
|
|
"num_tokens": 21345251.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.26680251955986023,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.2668025314807892,
|
|
"step": 4520,
|
|
"step_time": 16.978254450764506,
|
|
"zero_adv_skip_rate": 0.398
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 388.0,
|
|
"completions/max_terminated_length": 341.0,
|
|
"completions/mean_length": 191.675,
|
|
"completions/mean_terminated_length": 179.45893096923828,
|
|
"completions/min_length": 82.2,
|
|
"completions/min_terminated_length": 82.2,
|
|
"entropy": 0.38184894009360243,
|
|
"epoch": 0.6043223052294557,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 1.4375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0032160308212041853,
|
|
"num_tokens": 21388177.0,
|
|
"reward": 0.4625,
|
|
"reward_std": 0.269358703494072,
|
|
"rewards/correctness_reward/mean": 0.4625,
|
|
"rewards/correctness_reward/std": 0.269358703494072,
|
|
"step": 4530,
|
|
"step_time": 13.64972382048145,
|
|
"zero_adv_skip_rate": 0.3978
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 636.7,
|
|
"completions/max_terminated_length": 519.4,
|
|
"completions/mean_length": 233.5125,
|
|
"completions/mean_terminated_length": 211.3625030517578,
|
|
"completions/min_length": 95.9,
|
|
"completions/min_terminated_length": 95.9,
|
|
"entropy": 0.27361141946166756,
|
|
"epoch": 0.6056563500533618,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.06291443705558777,
|
|
"num_tokens": 21435186.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.21674493551254273,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.21674493551254273,
|
|
"step": 4540,
|
|
"step_time": 21.02880263356492,
|
|
"zero_adv_skip_rate": 0.398
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.2,
|
|
"completions/max_terminated_length": 300.2,
|
|
"completions/mean_length": 153.2375,
|
|
"completions/mean_terminated_length": 153.2375,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"entropy": 0.3871736720736538,
|
|
"epoch": 0.6069903948772679,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.796875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.07572614550590515,
|
|
"num_tokens": 21478917.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.33288776576519014,
|
|
"rewards/correctness_reward/mean": 0.45,
|
|
"rewards/correctness_reward/std": 0.3328877717256546,
|
|
"step": 4550,
|
|
"step_time": 11.154505050182342,
|
|
"zero_adv_skip_rate": 0.3978
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 536.5,
|
|
"completions/max_terminated_length": 351.0,
|
|
"completions/mean_length": 197.125,
|
|
"completions/mean_terminated_length": 172.76250076293945,
|
|
"completions/min_length": 76.6,
|
|
"completions/min_terminated_length": 76.6,
|
|
"entropy": 0.3324543996196654,
|
|
"epoch": 0.608324439701174,
|
|
"frac_reward_zero_std": 0.1,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.07748554348945617,
|
|
"num_tokens": 21523711.0,
|
|
"reward": 0.5625,
|
|
"reward_std": 0.4254619300365448,
|
|
"rewards/correctness_reward/mean": 0.5625,
|
|
"rewards/correctness_reward/std": 0.42546194791793823,
|
|
"step": 4560,
|
|
"step_time": 18.802217165194453,
|
|
"zero_adv_skip_rate": 0.3971
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 563.3,
|
|
"completions/max_terminated_length": 367.3,
|
|
"completions/mean_length": 200.375,
|
|
"completions/mean_terminated_length": 175.40357208251953,
|
|
"completions/min_length": 82.1,
|
|
"completions/min_terminated_length": 82.1,
|
|
"entropy": 0.33130257041193545,
|
|
"epoch": 0.6096584845250801,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.38671875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.13028922080993652,
|
|
"num_tokens": 21566789.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.15235702097415924,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.15235702097415924,
|
|
"step": 4570,
|
|
"step_time": 18.706612981110812,
|
|
"zero_adv_skip_rate": 0.3976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 277.1,
|
|
"completions/max_terminated_length": 276.9,
|
|
"completions/mean_length": 140.075,
|
|
"completions/mean_terminated_length": 139.80357208251954,
|
|
"completions/min_length": 80.7,
|
|
"completions/min_terminated_length": 80.7,
|
|
"entropy": 0.281703257933259,
|
|
"epoch": 0.6109925293489862,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.5390625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.07462955117225648,
|
|
"num_tokens": 21604643.0,
|
|
"reward": 0.5625,
|
|
"reward_std": 0.2558668524026871,
|
|
"rewards/correctness_reward/mean": 0.5625,
|
|
"rewards/correctness_reward/std": 0.25586686432361605,
|
|
"step": 4580,
|
|
"step_time": 10.255381480418146,
|
|
"zero_adv_skip_rate": 0.3976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 508.4,
|
|
"completions/max_terminated_length": 451.6,
|
|
"completions/mean_length": 256.1625,
|
|
"completions/mean_terminated_length": 238.35893058776855,
|
|
"completions/min_length": 100.5,
|
|
"completions/min_terminated_length": 100.5,
|
|
"entropy": 0.3177267462015152,
|
|
"epoch": 0.6123265741728922,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.62890625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04501877725124359,
|
|
"num_tokens": 21653328.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.321952098608017,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.32195210456848145,
|
|
"step": 4590,
|
|
"step_time": 17.579364260938018,
|
|
"zero_adv_skip_rate": 0.3974
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 717.3,
|
|
"completions/max_terminated_length": 616.0,
|
|
"completions/mean_length": 373.4375,
|
|
"completions/mean_terminated_length": 339.37916870117186,
|
|
"completions/min_length": 117.2,
|
|
"completions/min_terminated_length": 117.2,
|
|
"entropy": 0.30311315110884607,
|
|
"epoch": 0.6136606189967982,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.1982421875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.033028775453567506,
|
|
"num_tokens": 21714275.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.35561010539531707,
|
|
"rewards/correctness_reward/mean": 0.3375,
|
|
"rewards/correctness_reward/std": 0.35561011731624603,
|
|
"step": 4600,
|
|
"step_time": 24.361810854822398,
|
|
"zero_adv_skip_rate": 0.397
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 388.9,
|
|
"completions/max_terminated_length": 362.3,
|
|
"completions/mean_length": 227.075,
|
|
"completions/mean_terminated_length": 192.71488342285156,
|
|
"completions/min_length": 85.9,
|
|
"completions/min_terminated_length": 85.9,
|
|
"entropy": 0.40057697594165803,
|
|
"epoch": 0.6149946638207043,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.017718014121055604,
|
|
"num_tokens": 21764249.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.23144718110561371,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.23144719302654265,
|
|
"step": 4610,
|
|
"step_time": 13.68611463336274,
|
|
"zero_adv_skip_rate": 0.3972
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 547.4,
|
|
"completions/max_terminated_length": 362.8,
|
|
"completions/mean_length": 207.0875,
|
|
"completions/mean_terminated_length": 181.60714416503907,
|
|
"completions/min_length": 94.7,
|
|
"completions/min_terminated_length": 94.7,
|
|
"entropy": 0.2980057567358017,
|
|
"epoch": 0.6163287086446104,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.031406262516975404,
|
|
"num_tokens": 21810936.0,
|
|
"reward": 0.3875,
|
|
"reward_std": 0.2512414216995239,
|
|
"rewards/correctness_reward/mean": 0.3875,
|
|
"rewards/correctness_reward/std": 0.2512414276599884,
|
|
"step": 4620,
|
|
"step_time": 18.69537526052445,
|
|
"zero_adv_skip_rate": 0.3974
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 627.9,
|
|
"completions/max_terminated_length": 501.2,
|
|
"completions/mean_length": 234.75,
|
|
"completions/mean_terminated_length": 213.01785888671876,
|
|
"completions/min_length": 86.9,
|
|
"completions/min_terminated_length": 86.9,
|
|
"entropy": 0.3551865105206768,
|
|
"epoch": 0.6176627534685165,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.008793151378631592,
|
|
"num_tokens": 21856580.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.26302809417247774,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.26302810609340666,
|
|
"step": 4630,
|
|
"step_time": 21.47093530371785,
|
|
"zero_adv_skip_rate": 0.3974
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 596.1,
|
|
"completions/max_terminated_length": 254.5,
|
|
"completions/mean_length": 177.6875,
|
|
"completions/mean_terminated_length": 130.22857208251952,
|
|
"completions/min_length": 65.1,
|
|
"completions/min_terminated_length": 65.1,
|
|
"entropy": 0.3394403038546443,
|
|
"epoch": 0.6189967982924226,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 1.2421875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.22931966781616211,
|
|
"num_tokens": 21899179.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.2630359411239624,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.2630359411239624,
|
|
"step": 4640,
|
|
"step_time": 20.462582853715865,
|
|
"zero_adv_skip_rate": 0.3974
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 694.7,
|
|
"completions/max_terminated_length": 327.1,
|
|
"completions/mean_length": 231.0,
|
|
"completions/mean_terminated_length": 182.89643096923828,
|
|
"completions/min_length": 99.2,
|
|
"completions/min_terminated_length": 99.2,
|
|
"entropy": 0.3442561272531748,
|
|
"epoch": 0.6203308431163287,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.494140625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0009370770305395127,
|
|
"num_tokens": 21945635.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.18515617549419403,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.185156187415123,
|
|
"step": 4650,
|
|
"step_time": 22.74379921583459,
|
|
"zero_adv_skip_rate": 0.3978
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 980.4,
|
|
"completions/max_terminated_length": 497.1,
|
|
"completions/mean_length": 333.1375,
|
|
"completions/mean_terminated_length": 250.5196502685547,
|
|
"completions/min_length": 105.5,
|
|
"completions/min_terminated_length": 105.5,
|
|
"entropy": 0.39260873493428033,
|
|
"epoch": 0.6216648879402348,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.31640625,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.019529403746128084,
|
|
"num_tokens": 22001926.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.2558668524026871,
|
|
"rewards/correctness_reward/mean": 0.2625,
|
|
"rewards/correctness_reward/std": 0.25586686432361605,
|
|
"step": 4660,
|
|
"step_time": 32.35488156620413,
|
|
"zero_adv_skip_rate": 0.3979
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 632.8,
|
|
"completions/max_terminated_length": 441.7,
|
|
"completions/mean_length": 247.6625,
|
|
"completions/mean_terminated_length": 222.25357208251953,
|
|
"completions/min_length": 102.9,
|
|
"completions/min_terminated_length": 102.9,
|
|
"entropy": 0.3570783647398154,
|
|
"epoch": 0.6229989327641409,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 1.4921875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.08637991547584534,
|
|
"num_tokens": 22051771.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.2794355183839798,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.2794355243444443,
|
|
"step": 4670,
|
|
"step_time": 21.487099520489572,
|
|
"zero_adv_skip_rate": 0.3979
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 362.4,
|
|
"completions/max_terminated_length": 256.3,
|
|
"completions/mean_length": 156.95,
|
|
"completions/mean_terminated_length": 136.69643173217773,
|
|
"completions/min_length": 77.9,
|
|
"completions/min_terminated_length": 77.9,
|
|
"entropy": 0.3224756197471704,
|
|
"epoch": 0.624332977588047,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.70703125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0873576283454895,
|
|
"num_tokens": 22092543.0,
|
|
"reward": 0.5,
|
|
"reward_std": 0.2584230363368988,
|
|
"rewards/correctness_reward/mean": 0.5,
|
|
"rewards/correctness_reward/std": 0.2584230363368988,
|
|
"step": 4680,
|
|
"step_time": 12.858100096229464,
|
|
"zero_adv_skip_rate": 0.3976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 428.0,
|
|
"completions/max_terminated_length": 328.2,
|
|
"completions/mean_length": 196.05,
|
|
"completions/mean_terminated_length": 177.66428604125977,
|
|
"completions/min_length": 80.9,
|
|
"completions/min_terminated_length": 80.9,
|
|
"entropy": 0.3862619582563639,
|
|
"epoch": 0.625667022411953,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.017886289954185487,
|
|
"num_tokens": 22135331.0,
|
|
"reward": 0.475,
|
|
"reward_std": 0.22051151394844054,
|
|
"rewards/correctness_reward/mean": 0.475,
|
|
"rewards/correctness_reward/std": 0.2205115258693695,
|
|
"step": 4690,
|
|
"step_time": 14.799611263535917,
|
|
"zero_adv_skip_rate": 0.3979
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 1159.3,
|
|
"completions/max_terminated_length": 532.8,
|
|
"completions/mean_length": 336.4875,
|
|
"completions/mean_terminated_length": 242.56250762939453,
|
|
"completions/min_length": 91.4,
|
|
"completions/min_terminated_length": 91.4,
|
|
"entropy": 0.4153507133679731,
|
|
"epoch": 0.6270010672358591,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.3125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.16156290769577025,
|
|
"num_tokens": 22190930.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.3130935251712799,
|
|
"rewards/correctness_reward/mean": 0.225,
|
|
"rewards/correctness_reward/std": 0.3130935370922089,
|
|
"step": 4700,
|
|
"step_time": 38.032637135405096,
|
|
"zero_adv_skip_rate": 0.3977
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 582.3,
|
|
"completions/max_terminated_length": 582.3,
|
|
"completions/mean_length": 234.975,
|
|
"completions/mean_terminated_length": 234.975,
|
|
"completions/min_length": 79.1,
|
|
"completions/min_terminated_length": 79.1,
|
|
"entropy": 0.30046599404886365,
|
|
"epoch": 0.6283351120597652,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.00035860724747180937,
|
|
"num_tokens": 22237064.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.17422835528850555,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.17422835528850555,
|
|
"step": 4710,
|
|
"step_time": 19.478804368246347,
|
|
"zero_adv_skip_rate": 0.3981
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 609.7,
|
|
"completions/max_terminated_length": 462.7,
|
|
"completions/mean_length": 268.325,
|
|
"completions/mean_terminated_length": 227.00774078369142,
|
|
"completions/min_length": 86.3,
|
|
"completions/min_terminated_length": 86.3,
|
|
"entropy": 0.3526177542018039,
|
|
"epoch": 0.6296691568836713,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.019255411624908448,
|
|
"num_tokens": 22286290.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.3257265239953995,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.32572652995586393,
|
|
"step": 4720,
|
|
"step_time": 20.903625566605477,
|
|
"zero_adv_skip_rate": 0.3979
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 522.1,
|
|
"completions/max_terminated_length": 492.9,
|
|
"completions/mean_length": 247.8,
|
|
"completions/mean_terminated_length": 229.90178985595702,
|
|
"completions/min_length": 102.1,
|
|
"completions/min_terminated_length": 102.1,
|
|
"entropy": 0.3714020662009716,
|
|
"epoch": 0.6310032017075774,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.984375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.03003608584403992,
|
|
"num_tokens": 22339946.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.361081862449646,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.3610818684101105,
|
|
"step": 4730,
|
|
"step_time": 18.37547821579501,
|
|
"zero_adv_skip_rate": 0.3975
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 944.4,
|
|
"completions/max_terminated_length": 517.3,
|
|
"completions/mean_length": 271.975,
|
|
"completions/mean_terminated_length": 206.30178833007812,
|
|
"completions/min_length": 92.0,
|
|
"completions/min_terminated_length": 92.0,
|
|
"entropy": 0.4528558734804392,
|
|
"epoch": 0.6323372465314835,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1351334571838379,
|
|
"num_tokens": 22391432.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.20958369374275207,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.20958369374275207,
|
|
"step": 4740,
|
|
"step_time": 31.131067180354147,
|
|
"zero_adv_skip_rate": 0.3977
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 386.5,
|
|
"completions/max_terminated_length": 378.7,
|
|
"completions/mean_length": 177.0375,
|
|
"completions/mean_terminated_length": 174.3178581237793,
|
|
"completions/min_length": 98.0,
|
|
"completions/min_terminated_length": 98.0,
|
|
"entropy": 0.4366294307401404,
|
|
"epoch": 0.6336712913553896,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.88671875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0025917373597621916,
|
|
"num_tokens": 22435723.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.3337387710809708,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.33373878300189974,
|
|
"step": 4750,
|
|
"step_time": 14.050457026902587,
|
|
"zero_adv_skip_rate": 0.3973
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 691.0,
|
|
"completions/max_terminated_length": 486.7,
|
|
"completions/mean_length": 255.025,
|
|
"completions/mean_terminated_length": 212.43750381469727,
|
|
"completions/min_length": 78.7,
|
|
"completions/min_terminated_length": 78.7,
|
|
"entropy": 0.319993843169262,
|
|
"epoch": 0.6350053361792957,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.05416831374168396,
|
|
"num_tokens": 22484229.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.2575641840696335,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.25756419003009795,
|
|
"step": 4760,
|
|
"step_time": 23.290942866541446,
|
|
"zero_adv_skip_rate": 0.3973
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 853.9,
|
|
"completions/max_terminated_length": 703.0,
|
|
"completions/mean_length": 361.925,
|
|
"completions/mean_terminated_length": 315.2630981445312,
|
|
"completions/min_length": 132.3,
|
|
"completions/min_terminated_length": 132.3,
|
|
"entropy": 0.3209081808371203,
|
|
"epoch": 0.6363393810032018,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.333984375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.08256668448448182,
|
|
"num_tokens": 22543111.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.2802865236997604,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.2802865356206894,
|
|
"step": 4770,
|
|
"step_time": 28.714178405422718,
|
|
"zero_adv_skip_rate": 0.3971
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 614.3,
|
|
"completions/max_terminated_length": 465.6,
|
|
"completions/mean_length": 237.275,
|
|
"completions/mean_terminated_length": 215.5,
|
|
"completions/min_length": 115.1,
|
|
"completions/min_terminated_length": 115.1,
|
|
"entropy": 0.3145601366025706,
|
|
"epoch": 0.6376734258271078,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.02397126257419586,
|
|
"num_tokens": 22589661.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.29413776397705077,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.2941377818584442,
|
|
"step": 4780,
|
|
"step_time": 20.85490963263437,
|
|
"zero_adv_skip_rate": 0.3971
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 593.3,
|
|
"completions/max_terminated_length": 471.5,
|
|
"completions/mean_length": 297.725,
|
|
"completions/mean_terminated_length": 237.28250122070312,
|
|
"completions/min_length": 106.3,
|
|
"completions/min_terminated_length": 106.3,
|
|
"entropy": 0.31370738334953785,
|
|
"epoch": 0.6390074706510138,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.58984375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.008454772084951401,
|
|
"num_tokens": 22640679.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.2748226135969162,
|
|
"rewards/correctness_reward/mean": 0.25,
|
|
"rewards/correctness_reward/std": 0.2748226195573807,
|
|
"step": 4790,
|
|
"step_time": 20.344934868905693,
|
|
"zero_adv_skip_rate": 0.3969
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 799.8,
|
|
"completions/max_terminated_length": 608.3,
|
|
"completions/mean_length": 300.9125,
|
|
"completions/mean_terminated_length": 269.1125061035156,
|
|
"completions/min_length": 113.9,
|
|
"completions/min_terminated_length": 113.9,
|
|
"entropy": 0.308102966286242,
|
|
"epoch": 0.6403415154749199,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.294921875,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.08979972600936889,
|
|
"num_tokens": 22693520.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.2095758467912674,
|
|
"rewards/correctness_reward/mean": 0.2875,
|
|
"rewards/correctness_reward/std": 0.20957585871219636,
|
|
"step": 4800,
|
|
"step_time": 26.418746364116668,
|
|
"zero_adv_skip_rate": 0.3971
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 487.2,
|
|
"completions/max_terminated_length": 487.2,
|
|
"completions/mean_length": 218.9625,
|
|
"completions/mean_terminated_length": 218.9625,
|
|
"completions/min_length": 107.0,
|
|
"completions/min_terminated_length": 107.0,
|
|
"entropy": 0.3770535489311442,
|
|
"epoch": 0.641675560298826,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 0.70703125,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1269509196281433,
|
|
"num_tokens": 22738773.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.3282748609781265,
|
|
"rewards/correctness_reward/mean": 0.375,
|
|
"rewards/correctness_reward/std": 0.328274866938591,
|
|
"step": 4810,
|
|
"step_time": 17.066445564851165,
|
|
"zero_adv_skip_rate": 0.3967
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 591.3,
|
|
"completions/max_terminated_length": 406.6,
|
|
"completions/mean_length": 231.6625,
|
|
"completions/mean_terminated_length": 182.4095260620117,
|
|
"completions/min_length": 82.8,
|
|
"completions/min_terminated_length": 82.8,
|
|
"entropy": 0.3849782235920429,
|
|
"epoch": 0.6430096051227321,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.029490360617637636,
|
|
"num_tokens": 22784322.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.2975324273109436,
|
|
"rewards/correctness_reward/mean": 0.3375,
|
|
"rewards/correctness_reward/std": 0.2975324332714081,
|
|
"step": 4820,
|
|
"step_time": 20.302774036489428,
|
|
"zero_adv_skip_rate": 0.3967
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 515.0,
|
|
"completions/max_terminated_length": 515.0,
|
|
"completions/mean_length": 190.8875,
|
|
"completions/mean_terminated_length": 190.8875,
|
|
"completions/min_length": 76.0,
|
|
"completions/min_terminated_length": 76.0,
|
|
"entropy": 0.3247316942683288,
|
|
"epoch": 0.6443436499466382,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.427734375,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.001730450615286827,
|
|
"num_tokens": 22827825.0,
|
|
"reward": 0.4375,
|
|
"reward_std": 0.3622844099998474,
|
|
"rewards/correctness_reward/mean": 0.4375,
|
|
"rewards/correctness_reward/std": 0.36228445172309875,
|
|
"step": 4830,
|
|
"step_time": 18.373824126273394,
|
|
"zero_adv_skip_rate": 0.3965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 585.1,
|
|
"completions/max_terminated_length": 552.9,
|
|
"completions/mean_length": 297.8875,
|
|
"completions/mean_terminated_length": 280.49464416503906,
|
|
"completions/min_length": 104.5,
|
|
"completions/min_terminated_length": 104.5,
|
|
"entropy": 0.33648872235789895,
|
|
"epoch": 0.6456776947705443,
|
|
"frac_reward_zero_std": 0.2,
|
|
"grad_norm": 1.234375,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.06889815330505371,
|
|
"num_tokens": 22880192.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.3737070143222809,
|
|
"rewards/correctness_reward/mean": 0.45,
|
|
"rewards/correctness_reward/std": 0.37370702624320984,
|
|
"step": 4840,
|
|
"step_time": 20.737340954784305,
|
|
"zero_adv_skip_rate": 0.3961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 549.4,
|
|
"completions/max_terminated_length": 507.3,
|
|
"completions/mean_length": 234.625,
|
|
"completions/mean_terminated_length": 227.65893020629883,
|
|
"completions/min_length": 97.5,
|
|
"completions/min_terminated_length": 97.5,
|
|
"entropy": 0.39588422964637476,
|
|
"epoch": 0.6470117395944504,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.6171875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.10280247926712036,
|
|
"num_tokens": 22929482.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.24662851691246032,
|
|
"rewards/correctness_reward/mean": 0.1625,
|
|
"rewards/correctness_reward/std": 0.2466285228729248,
|
|
"step": 4850,
|
|
"step_time": 19.493375701177865,
|
|
"zero_adv_skip_rate": 0.3961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 615.0,
|
|
"completions/max_terminated_length": 298.6,
|
|
"completions/mean_length": 199.2625,
|
|
"completions/mean_terminated_length": 155.9857162475586,
|
|
"completions/min_length": 84.6,
|
|
"completions/min_terminated_length": 84.6,
|
|
"entropy": 0.26365462969988585,
|
|
"epoch": 0.6483457844183564,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0047841183841228485,
|
|
"num_tokens": 22971471.0,
|
|
"reward": 0.525,
|
|
"reward_std": 0.15782093107700348,
|
|
"rewards/correctness_reward/mean": 0.525,
|
|
"rewards/correctness_reward/std": 0.15782093703746797,
|
|
"step": 4860,
|
|
"step_time": 21.26041017854586,
|
|
"zero_adv_skip_rate": 0.3965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 474.9,
|
|
"completions/max_terminated_length": 283.2,
|
|
"completions/mean_length": 183.3875,
|
|
"completions/mean_terminated_length": 158.85893020629882,
|
|
"completions/min_length": 81.1,
|
|
"completions/min_terminated_length": 81.1,
|
|
"entropy": 0.26739286072552204,
|
|
"epoch": 0.6496798292422625,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.11703609228134156,
|
|
"num_tokens": 23014454.0,
|
|
"reward": 0.45,
|
|
"reward_std": 0.2722742766141891,
|
|
"rewards/correctness_reward/mean": 0.45,
|
|
"rewards/correctness_reward/std": 0.27227428257465364,
|
|
"step": 4870,
|
|
"step_time": 17.05254058353603,
|
|
"zero_adv_skip_rate": 0.3965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 1133.8,
|
|
"completions/max_terminated_length": 494.5,
|
|
"completions/mean_length": 316.4375,
|
|
"completions/mean_terminated_length": 224.49821853637695,
|
|
"completions/min_length": 81.1,
|
|
"completions/min_terminated_length": 81.1,
|
|
"entropy": 0.3099472452221172,
|
|
"epoch": 0.6510138740661686,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.40625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.1158103108406067,
|
|
"num_tokens": 23074153.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.3202547669410706,
|
|
"rewards/correctness_reward/mean": 0.4,
|
|
"rewards/correctness_reward/std": 0.3202547788619995,
|
|
"step": 4880,
|
|
"step_time": 38.611833308637145,
|
|
"zero_adv_skip_rate": 0.3963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 605.3,
|
|
"completions/max_terminated_length": 437.7,
|
|
"completions/mean_length": 228.85,
|
|
"completions/mean_terminated_length": 188.9232177734375,
|
|
"completions/min_length": 72.2,
|
|
"completions/min_terminated_length": 72.2,
|
|
"entropy": 0.34326816148435074,
|
|
"epoch": 0.6523479188900747,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.19140625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.015681920945644377,
|
|
"num_tokens": 23122357.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.2558668524026871,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.25586686432361605,
|
|
"step": 4890,
|
|
"step_time": 21.34696087986231,
|
|
"zero_adv_skip_rate": 0.3963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 602.3,
|
|
"completions/max_terminated_length": 396.0,
|
|
"completions/mean_length": 230.1875,
|
|
"completions/mean_terminated_length": 196.21607398986816,
|
|
"completions/min_length": 117.2,
|
|
"completions/min_terminated_length": 117.2,
|
|
"entropy": 0.29098060860165526,
|
|
"epoch": 0.6536819637139808,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.3203125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.04112994968891144,
|
|
"num_tokens": 23168500.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.3202547669410706,
|
|
"rewards/correctness_reward/mean": 0.275,
|
|
"rewards/correctness_reward/std": 0.3202547788619995,
|
|
"step": 4900,
|
|
"step_time": 21.240406797360627,
|
|
"zero_adv_skip_rate": 0.3961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 780.6,
|
|
"completions/max_terminated_length": 600.8,
|
|
"completions/mean_length": 277.2875,
|
|
"completions/mean_terminated_length": 250.7339309692383,
|
|
"completions/min_length": 103.1,
|
|
"completions/min_terminated_length": 103.1,
|
|
"entropy": 0.38549620285630226,
|
|
"epoch": 0.6550160085378869,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.2275390625,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.034107100963592527,
|
|
"num_tokens": 23225835.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.17422050833702088,
|
|
"rewards/correctness_reward/mean": 0.1,
|
|
"rewards/correctness_reward/std": 0.17422052025794982,
|
|
"step": 4910,
|
|
"step_time": 26.60901929186657,
|
|
"zero_adv_skip_rate": 0.3965
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.075,
|
|
"completions/max_length": 1076.9,
|
|
"completions/max_terminated_length": 455.8,
|
|
"completions/mean_length": 289.775,
|
|
"completions/mean_terminated_length": 197.97500228881836,
|
|
"completions/min_length": 80.3,
|
|
"completions/min_terminated_length": 80.3,
|
|
"entropy": 0.47924742608198095,
|
|
"epoch": 0.656350053361793,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.71875,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.08401397466659546,
|
|
"num_tokens": 23280785.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.31479085683822633,
|
|
"rewards/correctness_reward/mean": 0.1875,
|
|
"rewards/correctness_reward/std": 0.3147908627986908,
|
|
"step": 4920,
|
|
"step_time": 36.73750554183498,
|
|
"zero_adv_skip_rate": 0.3963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 395.2,
|
|
"completions/max_terminated_length": 395.2,
|
|
"completions/mean_length": 190.3,
|
|
"completions/mean_terminated_length": 190.3,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"entropy": 0.36180410467620405,
|
|
"epoch": 0.6576840981856991,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.7578125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.09234949350357055,
|
|
"num_tokens": 23326353.0,
|
|
"reward": 0.55,
|
|
"reward_std": 0.28029437065124513,
|
|
"rewards/correctness_reward/mean": 0.55,
|
|
"rewards/correctness_reward/std": 0.28029437065124513,
|
|
"step": 4930,
|
|
"step_time": 14.570873779058456,
|
|
"zero_adv_skip_rate": 0.3961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 522.5,
|
|
"completions/max_terminated_length": 493.0,
|
|
"completions/mean_length": 214.5625,
|
|
"completions/mean_terminated_length": 209.56250076293946,
|
|
"completions/min_length": 82.1,
|
|
"completions/min_terminated_length": 82.1,
|
|
"entropy": 0.3845489326243599,
|
|
"epoch": 0.6590181430096052,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.06993361115455628,
|
|
"num_tokens": 23370870.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/correctness_reward/mean": 0.175,
|
|
"rewards/correctness_reward/std": 0.2504029482603073,
|
|
"step": 4940,
|
|
"step_time": 18.33775127502158,
|
|
"zero_adv_skip_rate": 0.3962
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 622.4,
|
|
"completions/max_terminated_length": 316.5,
|
|
"completions/mean_length": 225.85,
|
|
"completions/mean_terminated_length": 182.10357589721679,
|
|
"completions/min_length": 88.1,
|
|
"completions/min_terminated_length": 88.1,
|
|
"entropy": 0.29995624007036287,
|
|
"epoch": 0.6603521878335112,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.030384179949760438,
|
|
"num_tokens": 23415138.0,
|
|
"reward": 0.425,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/correctness_reward/mean": 0.425,
|
|
"rewards/correctness_reward/std": 0.2504029482603073,
|
|
"step": 4950,
|
|
"step_time": 21.665196518227457,
|
|
"zero_adv_skip_rate": 0.3962
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 556.1,
|
|
"completions/max_terminated_length": 547.0,
|
|
"completions/mean_length": 246.525,
|
|
"completions/mean_terminated_length": 244.04107208251952,
|
|
"completions/min_length": 88.4,
|
|
"completions/min_terminated_length": 88.4,
|
|
"entropy": 0.3787304969504476,
|
|
"epoch": 0.6616862326574173,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.01316453069448471,
|
|
"num_tokens": 23465124.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.21297051012516022,
|
|
"rewards/correctness_reward/mean": 0.3125,
|
|
"rewards/correctness_reward/std": 0.21297051012516022,
|
|
"step": 4960,
|
|
"step_time": 19.23376855375245,
|
|
"zero_adv_skip_rate": 0.3964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 444.5,
|
|
"completions/max_terminated_length": 442.2,
|
|
"completions/mean_length": 219.1875,
|
|
"completions/mean_terminated_length": 217.17678680419922,
|
|
"completions/min_length": 92.3,
|
|
"completions/min_terminated_length": 92.3,
|
|
"entropy": 0.3621425319773455,
|
|
"epoch": 0.6630202774813234,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.01531466245651245,
|
|
"num_tokens": 23524027.0,
|
|
"reward": 0.3625,
|
|
"reward_std": 0.29413776397705077,
|
|
"rewards/correctness_reward/mean": 0.3625,
|
|
"rewards/correctness_reward/std": 0.2941377818584442,
|
|
"step": 4970,
|
|
"step_time": 16.100848561991008,
|
|
"zero_adv_skip_rate": 0.3964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 500.1,
|
|
"completions/max_terminated_length": 500.1,
|
|
"completions/mean_length": 190.55,
|
|
"completions/mean_terminated_length": 190.55,
|
|
"completions/min_length": 75.0,
|
|
"completions/min_terminated_length": 75.0,
|
|
"entropy": 0.3149289855112632,
|
|
"epoch": 0.6643543223052295,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0075941398739814755,
|
|
"num_tokens": 23568823.0,
|
|
"reward": 0.5125,
|
|
"reward_std": 0.2886660069227219,
|
|
"rewards/correctness_reward/mean": 0.5125,
|
|
"rewards/correctness_reward/std": 0.28866603076457975,
|
|
"step": 4980,
|
|
"step_time": 18.02074984004721,
|
|
"zero_adv_skip_rate": 0.3964
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 564.8,
|
|
"completions/max_terminated_length": 564.8,
|
|
"completions/mean_length": 302.725,
|
|
"completions/mean_terminated_length": 302.725,
|
|
"completions/min_length": 137.7,
|
|
"completions/min_terminated_length": 137.7,
|
|
"entropy": 0.35495875170454383,
|
|
"epoch": 0.6656883671291356,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.04996286034584045,
|
|
"num_tokens": 23622337.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.19231741726398469,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.19231742918491362,
|
|
"step": 4990,
|
|
"step_time": 19.73052594633773,
|
|
"zero_adv_skip_rate": 0.3968
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 316.0,
|
|
"completions/max_terminated_length": 308.6,
|
|
"completions/mean_length": 141.0,
|
|
"completions/mean_terminated_length": 137.36250228881835,
|
|
"completions/min_length": 52.7,
|
|
"completions/min_terminated_length": 52.7,
|
|
"entropy": 0.3848136343682806,
|
|
"epoch": 0.6670224119530416,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.021188779175281523,
|
|
"num_tokens": 23662145.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.25210027396678925,
|
|
"rewards/correctness_reward/mean": 0.4125,
|
|
"rewards/correctness_reward/std": 0.25210027396678925,
|
|
"step": 5000,
|
|
"step_time": 11.751490045432002,
|
|
"zero_adv_skip_rate": 0.3968
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 7496,
|
|
"num_input_tokens_seen": 23662145,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 500,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|