2222 lines
78 KiB
JSON
2222 lines
78 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.08,
|
|
"eval_steps": 500,
|
|
"global_step": 800,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 382.0,
|
|
"completions/max_terminated_length": 382.0,
|
|
"completions/mean_length": 287.875,
|
|
"completions/mean_terminated_length": 287.875,
|
|
"completions/min_length": 203.0,
|
|
"completions/min_terminated_length": 203.0,
|
|
"entropy": 0.1642230749130249,
|
|
"epoch": 0.0001,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.665309488773346,
|
|
"learning_rate": 1e-05,
|
|
"loss": -0.11556684970855713,
|
|
"num_tokens": 3951.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.3535533845424652,
|
|
"rewards/boxed_accuracy_reward/mean": 0.125,
|
|
"rewards/boxed_accuracy_reward/std": 0.3535533845424652,
|
|
"step": 1,
|
|
"step_time": 26.083515274000092
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 438.1111111111111,
|
|
"completions/max_terminated_length": 438.1111111111111,
|
|
"completions/mean_length": 253.38888888888889,
|
|
"completions/mean_terminated_length": 253.38888888888889,
|
|
"completions/min_length": 132.33333333333334,
|
|
"completions/min_terminated_length": 132.33333333333334,
|
|
"entropy": 0.19083112478256226,
|
|
"epoch": 0.001,
|
|
"frac_reward_zero_std": 0.3333333333333333,
|
|
"grad_norm": 0.7672234177589417,
|
|
"learning_rate": 9.91e-06,
|
|
"loss": 0.017603478497929044,
|
|
"num_tokens": 34851.0,
|
|
"reward": 0.1527777777777778,
|
|
"reward_std": 0.2903762327300178,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1527777777777778,
|
|
"rewards/boxed_accuracy_reward/std": 0.2903762393527561,
|
|
"step": 10,
|
|
"step_time": 29.407547047666917
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 598.4,
|
|
"completions/max_terminated_length": 454.2,
|
|
"completions/mean_length": 326.7125,
|
|
"completions/mean_terminated_length": 281.9235778808594,
|
|
"completions/min_length": 157.3,
|
|
"completions/min_terminated_length": 157.3,
|
|
"entropy": 0.22966246232390403,
|
|
"epoch": 0.002,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 9.810000000000001e-06,
|
|
"loss": 0.015656116604804992,
|
|
"num_tokens": 74804.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.12416292428970337,
|
|
"rewards/boxed_accuracy_reward/mean": 0.15,
|
|
"rewards/boxed_accuracy_reward/std": 0.12416292428970337,
|
|
"step": 20,
|
|
"step_time": 40.079171687399864
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 471.4,
|
|
"completions/max_terminated_length": 418.5,
|
|
"completions/mean_length": 261.45,
|
|
"completions/mean_terminated_length": 251.90892944335937,
|
|
"completions/min_length": 161.7,
|
|
"completions/min_terminated_length": 161.7,
|
|
"entropy": 0.17057178877294063,
|
|
"epoch": 0.003,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 9.71e-06,
|
|
"loss": -0.01473381370306015,
|
|
"num_tokens": 109952.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.08711025416851044,
|
|
"rewards/boxed_accuracy_reward/mean": 0.25,
|
|
"rewards/boxed_accuracy_reward/std": 0.08711026012897491,
|
|
"step": 30,
|
|
"step_time": 31.344073937199575
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 462.8,
|
|
"completions/max_terminated_length": 416.3,
|
|
"completions/mean_length": 292.6375,
|
|
"completions/mean_terminated_length": 267.9024993896484,
|
|
"completions/min_length": 172.9,
|
|
"completions/min_terminated_length": 172.9,
|
|
"entropy": 0.22824809104204177,
|
|
"epoch": 0.004,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 9.610000000000001e-06,
|
|
"loss": 0.014347019791603088,
|
|
"num_tokens": 148715.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.09258201122283935,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1,
|
|
"rewards/boxed_accuracy_reward/std": 0.09258201122283935,
|
|
"step": 40,
|
|
"step_time": 30.823996917499972
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 543.5,
|
|
"completions/max_terminated_length": 415.6,
|
|
"completions/mean_length": 269.2375,
|
|
"completions/mean_terminated_length": 249.4482162475586,
|
|
"completions/min_length": 160.6,
|
|
"completions/min_terminated_length": 160.6,
|
|
"entropy": 0.266183939576149,
|
|
"epoch": 0.005,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.7730127573013306,
|
|
"learning_rate": 9.51e-06,
|
|
"loss": 0.047160887718200685,
|
|
"num_tokens": 184662.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.26849985122680664,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1625,
|
|
"rewards/boxed_accuracy_reward/std": 0.2684998571872711,
|
|
"step": 50,
|
|
"step_time": 36.26071728879979
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 445.4,
|
|
"completions/max_terminated_length": 445.4,
|
|
"completions/mean_length": 271.05,
|
|
"completions/mean_terminated_length": 271.05,
|
|
"completions/min_length": 168.9,
|
|
"completions/min_terminated_length": 168.9,
|
|
"entropy": 0.18989864736795425,
|
|
"epoch": 0.006,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 9.41e-06,
|
|
"loss": 0.07637788653373719,
|
|
"num_tokens": 221874.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.2276727557182312,
|
|
"rewards/boxed_accuracy_reward/mean": 0.25,
|
|
"rewards/boxed_accuracy_reward/std": 0.22767276763916017,
|
|
"step": 60,
|
|
"step_time": 29.8493166889999
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 559.9,
|
|
"completions/max_terminated_length": 456.6,
|
|
"completions/mean_length": 279.3125,
|
|
"completions/mean_terminated_length": 262.7035736083984,
|
|
"completions/min_length": 157.0,
|
|
"completions/min_terminated_length": 157.0,
|
|
"entropy": 0.2535190790891647,
|
|
"epoch": 0.007,
|
|
"frac_reward_zero_std": 0.9,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 9.31e-06,
|
|
"loss": 0.007473611086606979,
|
|
"num_tokens": 257171.0,
|
|
"reward": 0.05,
|
|
"reward_std": 0.05345224738121033,
|
|
"rewards/boxed_accuracy_reward/mean": 0.05,
|
|
"rewards/boxed_accuracy_reward/std": 0.05345224738121033,
|
|
"step": 70,
|
|
"step_time": 37.422609474000275
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 506.5,
|
|
"completions/max_terminated_length": 375.0,
|
|
"completions/mean_length": 246.025,
|
|
"completions/mean_terminated_length": 226.50714569091798,
|
|
"completions/min_length": 146.6,
|
|
"completions/min_terminated_length": 146.6,
|
|
"entropy": 0.21952729374170304,
|
|
"epoch": 0.008,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 9.210000000000002e-06,
|
|
"loss": -0.004495994746685028,
|
|
"num_tokens": 291301.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.18685350716114044,
|
|
"rewards/boxed_accuracy_reward/mean": 0.175,
|
|
"rewards/boxed_accuracy_reward/std": 0.18685351312160492,
|
|
"step": 80,
|
|
"step_time": 33.85629228980069
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 417.1,
|
|
"completions/max_terminated_length": 417.1,
|
|
"completions/mean_length": 270.3375,
|
|
"completions/mean_terminated_length": 270.3375,
|
|
"completions/min_length": 177.8,
|
|
"completions/min_terminated_length": 177.8,
|
|
"entropy": 0.2391076095402241,
|
|
"epoch": 0.009,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6646445393562317,
|
|
"learning_rate": 9.110000000000001e-06,
|
|
"loss": 0.0037924151867628096,
|
|
"num_tokens": 326552.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.2095758467912674,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1125,
|
|
"rewards/boxed_accuracy_reward/std": 0.20957585871219636,
|
|
"step": 90,
|
|
"step_time": 27.908505431900267
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 489.8,
|
|
"completions/max_terminated_length": 405.3,
|
|
"completions/mean_length": 305.5,
|
|
"completions/mean_terminated_length": 269.0066680908203,
|
|
"completions/min_length": 142.9,
|
|
"completions/min_terminated_length": 142.9,
|
|
"entropy": 0.2832915961742401,
|
|
"epoch": 0.01,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 9.01e-06,
|
|
"loss": 0.01493358165025711,
|
|
"num_tokens": 364440.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.17045392990112304,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2,
|
|
"rewards/boxed_accuracy_reward/std": 0.17045392990112304,
|
|
"step": 100,
|
|
"step_time": 32.493894719798845
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 454.8,
|
|
"completions/max_terminated_length": 377.0,
|
|
"completions/mean_length": 258.625,
|
|
"completions/mean_terminated_length": 248.3375015258789,
|
|
"completions/min_length": 170.1,
|
|
"completions/min_terminated_length": 170.1,
|
|
"entropy": 0.2617956634610891,
|
|
"epoch": 0.011,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.29700157046318054,
|
|
"learning_rate": 8.910000000000001e-06,
|
|
"loss": 0.03313029408454895,
|
|
"num_tokens": 398914.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.22598327100276946,
|
|
"rewards/boxed_accuracy_reward/mean": 0.125,
|
|
"rewards/boxed_accuracy_reward/std": 0.22598327696323395,
|
|
"step": 110,
|
|
"step_time": 30.349950915500084
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 457.5,
|
|
"completions/max_terminated_length": 389.1,
|
|
"completions/mean_length": 265.4875,
|
|
"completions/mean_terminated_length": 255.85714416503907,
|
|
"completions/min_length": 164.2,
|
|
"completions/min_terminated_length": 164.2,
|
|
"entropy": 0.2188379220664501,
|
|
"epoch": 0.012,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 8.81e-06,
|
|
"loss": 0.02148536443710327,
|
|
"num_tokens": 434921.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.08880758583545685,
|
|
"rewards/boxed_accuracy_reward/mean": 0.0625,
|
|
"rewards/boxed_accuracy_reward/std": 0.08880758583545685,
|
|
"step": 120,
|
|
"step_time": 30.65767741319978
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 424.0,
|
|
"completions/max_terminated_length": 382.3,
|
|
"completions/mean_length": 237.0125,
|
|
"completions/mean_terminated_length": 228.27142944335938,
|
|
"completions/min_length": 139.3,
|
|
"completions/min_terminated_length": 139.3,
|
|
"entropy": 0.2461308181285858,
|
|
"epoch": 0.013,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 8.710000000000001e-06,
|
|
"loss": 0.013855263590812683,
|
|
"num_tokens": 468658.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.20411193668842315,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1,
|
|
"rewards/boxed_accuracy_reward/std": 0.20411194264888763,
|
|
"step": 130,
|
|
"step_time": 28.27658463789994
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 416.1,
|
|
"completions/max_terminated_length": 357.2,
|
|
"completions/mean_length": 233.2625,
|
|
"completions/mean_terminated_length": 223.3732147216797,
|
|
"completions/min_length": 130.1,
|
|
"completions/min_terminated_length": 130.1,
|
|
"entropy": 0.29308837205171584,
|
|
"epoch": 0.014,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 8.61e-06,
|
|
"loss": 0.0346070647239685,
|
|
"num_tokens": 500159.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.22768060266971588,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1875,
|
|
"rewards/boxed_accuracy_reward/std": 0.22768060266971588,
|
|
"step": 140,
|
|
"step_time": 27.783400436999727
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 503.8,
|
|
"completions/max_terminated_length": 503.8,
|
|
"completions/mean_length": 295.05,
|
|
"completions/mean_terminated_length": 295.05,
|
|
"completions/min_length": 184.6,
|
|
"completions/min_terminated_length": 184.6,
|
|
"entropy": 0.20557359904050826,
|
|
"epoch": 0.015,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 8.51e-06,
|
|
"loss": 0.03165164291858673,
|
|
"num_tokens": 537435.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.14433692693710326,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1625,
|
|
"rewards/boxed_accuracy_reward/std": 0.14433693289756774,
|
|
"step": 150,
|
|
"step_time": 33.749151340000026
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 504.0,
|
|
"completions/max_terminated_length": 405.2,
|
|
"completions/mean_length": 280.05,
|
|
"completions/mean_terminated_length": 261.85714569091795,
|
|
"completions/min_length": 158.9,
|
|
"completions/min_terminated_length": 158.9,
|
|
"entropy": 0.2418084606528282,
|
|
"epoch": 0.016,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.9073280096054077,
|
|
"learning_rate": 8.41e-06,
|
|
"loss": 0.029856789112091064,
|
|
"num_tokens": 573351.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.2794355183839798,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2,
|
|
"rewards/boxed_accuracy_reward/std": 0.2794355243444443,
|
|
"step": 160,
|
|
"step_time": 36.18624399920045
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 484.3,
|
|
"completions/max_terminated_length": 484.3,
|
|
"completions/mean_length": 281.0,
|
|
"completions/mean_terminated_length": 281.0,
|
|
"completions/min_length": 167.8,
|
|
"completions/min_terminated_length": 167.8,
|
|
"entropy": 0.21754518747329712,
|
|
"epoch": 0.017,
|
|
"frac_reward_zero_std": 0.9,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 8.31e-06,
|
|
"loss": 0.01904902458190918,
|
|
"num_tokens": 610279.0,
|
|
"reward": 0.125,
|
|
"reward_std": 0.046291005611419675,
|
|
"rewards/boxed_accuracy_reward/mean": 0.125,
|
|
"rewards/boxed_accuracy_reward/std": 0.046291005611419675,
|
|
"step": 170,
|
|
"step_time": 35.24356591649958
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 472.6,
|
|
"completions/max_terminated_length": 421.2,
|
|
"completions/mean_length": 269.1625,
|
|
"completions/mean_terminated_length": 260.52142944335935,
|
|
"completions/min_length": 159.9,
|
|
"completions/min_terminated_length": 159.9,
|
|
"entropy": 0.22995990961790086,
|
|
"epoch": 0.018,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 8.210000000000001e-06,
|
|
"loss": -0.011320264637470245,
|
|
"num_tokens": 646660.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.08880758583545685,
|
|
"rewards/boxed_accuracy_reward/mean": 0.0625,
|
|
"rewards/boxed_accuracy_reward/std": 0.08880758583545685,
|
|
"step": 180,
|
|
"step_time": 35.16620119949948
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 570.6,
|
|
"completions/max_terminated_length": 513.5,
|
|
"completions/mean_length": 297.2625,
|
|
"completions/mean_terminated_length": 272.977978515625,
|
|
"completions/min_length": 156.5,
|
|
"completions/min_terminated_length": 156.5,
|
|
"entropy": 0.24057478606700897,
|
|
"epoch": 0.019,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 1.0508952140808105,
|
|
"learning_rate": 8.110000000000002e-06,
|
|
"loss": 0.01611926555633545,
|
|
"num_tokens": 685673.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.1948736011981964,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1,
|
|
"rewards/boxed_accuracy_reward/std": 0.1948736011981964,
|
|
"step": 190,
|
|
"step_time": 39.78827415060077
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 379.3,
|
|
"completions/max_terminated_length": 322.7,
|
|
"completions/mean_length": 236.0875,
|
|
"completions/mean_terminated_length": 227.3767883300781,
|
|
"completions/min_length": 150.7,
|
|
"completions/min_terminated_length": 150.7,
|
|
"entropy": 0.21252653896808624,
|
|
"epoch": 0.02,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.7992851138114929,
|
|
"learning_rate": 8.010000000000001e-06,
|
|
"loss": 0.02353036105632782,
|
|
"num_tokens": 719336.0,
|
|
"reward": 0.35,
|
|
"reward_std": 0.30299633741378784,
|
|
"rewards/boxed_accuracy_reward/mean": 0.35,
|
|
"rewards/boxed_accuracy_reward/std": 0.3029963493347168,
|
|
"step": 200,
|
|
"step_time": 27.117062961000194
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0875,
|
|
"completions/max_length": 609.6,
|
|
"completions/max_terminated_length": 431.3,
|
|
"completions/mean_length": 345.1875,
|
|
"completions/mean_terminated_length": 294.5657165527344,
|
|
"completions/min_length": 165.6,
|
|
"completions/min_terminated_length": 165.6,
|
|
"entropy": 0.17211427241563798,
|
|
"epoch": 0.021,
|
|
"frac_reward_zero_std": 0.9,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 7.91e-06,
|
|
"loss": -0.0028946617618203162,
|
|
"num_tokens": 759991.0,
|
|
"reward": 0.0125,
|
|
"reward_std": 0.03535533845424652,
|
|
"rewards/boxed_accuracy_reward/mean": 0.0125,
|
|
"rewards/boxed_accuracy_reward/std": 0.03535533845424652,
|
|
"step": 210,
|
|
"step_time": 44.109834954999315
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 562.9,
|
|
"completions/max_terminated_length": 454.9,
|
|
"completions/mean_length": 275.2125,
|
|
"completions/mean_terminated_length": 257.24286193847655,
|
|
"completions/min_length": 149.9,
|
|
"completions/min_terminated_length": 149.9,
|
|
"entropy": 0.2036581441760063,
|
|
"epoch": 0.022,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.9508808255195618,
|
|
"learning_rate": 7.810000000000001e-06,
|
|
"loss": 0.05190147757530213,
|
|
"num_tokens": 795232.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.17422835528850555,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2375,
|
|
"rewards/boxed_accuracy_reward/std": 0.17422835528850555,
|
|
"step": 220,
|
|
"step_time": 38.12656000370043
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 372.2,
|
|
"completions/max_terminated_length": 322.1,
|
|
"completions/mean_length": 216.725,
|
|
"completions/mean_terminated_length": 206.85714416503907,
|
|
"completions/min_length": 123.2,
|
|
"completions/min_terminated_length": 123.2,
|
|
"entropy": 0.24748676642775536,
|
|
"epoch": 0.023,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.825386643409729,
|
|
"learning_rate": 7.71e-06,
|
|
"loss": -0.019961801171302796,
|
|
"num_tokens": 828634.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.1632926881313324,
|
|
"rewards/boxed_accuracy_reward/mean": 0.225,
|
|
"rewards/boxed_accuracy_reward/std": 0.1632926881313324,
|
|
"step": 230,
|
|
"step_time": 24.96606878720049
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 373.3,
|
|
"completions/max_terminated_length": 373.3,
|
|
"completions/mean_length": 232.2,
|
|
"completions/mean_terminated_length": 232.2,
|
|
"completions/min_length": 150.1,
|
|
"completions/min_terminated_length": 150.1,
|
|
"entropy": 0.2603755287826061,
|
|
"epoch": 0.024,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.5379490256309509,
|
|
"learning_rate": 7.610000000000001e-06,
|
|
"loss": 0.03033584952354431,
|
|
"num_tokens": 861162.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.2722664296627045,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1875,
|
|
"rewards/boxed_accuracy_reward/std": 0.2722664475440979,
|
|
"step": 240,
|
|
"step_time": 24.956236593399808
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 532.6,
|
|
"completions/max_terminated_length": 465.1,
|
|
"completions/mean_length": 262.9375,
|
|
"completions/mean_terminated_length": 252.74464416503906,
|
|
"completions/min_length": 135.1,
|
|
"completions/min_terminated_length": 135.1,
|
|
"entropy": 0.2639094963669777,
|
|
"epoch": 0.025,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 7.510000000000001e-06,
|
|
"loss": 0.014231480658054352,
|
|
"num_tokens": 896165.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.08711025416851044,
|
|
"rewards/boxed_accuracy_reward/mean": 0.075,
|
|
"rewards/boxed_accuracy_reward/std": 0.08711026012897491,
|
|
"step": 250,
|
|
"step_time": 35.73487980219979
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 416.4,
|
|
"completions/max_terminated_length": 416.4,
|
|
"completions/mean_length": 263.975,
|
|
"completions/mean_terminated_length": 263.975,
|
|
"completions/min_length": 163.7,
|
|
"completions/min_terminated_length": 163.7,
|
|
"entropy": 0.249638444930315,
|
|
"epoch": 0.026,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 7.41e-06,
|
|
"loss": 0.01857263296842575,
|
|
"num_tokens": 930771.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.15235702097415924,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1625,
|
|
"rewards/boxed_accuracy_reward/std": 0.15235702097415924,
|
|
"step": 260,
|
|
"step_time": 28.139153176400214
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 422.7,
|
|
"completions/max_terminated_length": 422.7,
|
|
"completions/mean_length": 251.9875,
|
|
"completions/mean_terminated_length": 251.9875,
|
|
"completions/min_length": 153.3,
|
|
"completions/min_terminated_length": 153.3,
|
|
"entropy": 0.24129935801029206,
|
|
"epoch": 0.027,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 7.31e-06,
|
|
"loss": -0.032139381766319274,
|
|
"num_tokens": 963458.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.14433692693710326,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2125,
|
|
"rewards/boxed_accuracy_reward/std": 0.14433693289756774,
|
|
"step": 270,
|
|
"step_time": 28.544226764599806
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 332.8,
|
|
"completions/max_terminated_length": 332.8,
|
|
"completions/mean_length": 231.9125,
|
|
"completions/mean_terminated_length": 231.9125,
|
|
"completions/min_length": 148.2,
|
|
"completions/min_terminated_length": 148.2,
|
|
"entropy": 0.26727280877530574,
|
|
"epoch": 0.028,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.9024311900138855,
|
|
"learning_rate": 7.2100000000000004e-06,
|
|
"loss": 0.021772387623786926,
|
|
"num_tokens": 996579.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.17422050833702088,
|
|
"rewards/boxed_accuracy_reward/mean": 0.15,
|
|
"rewards/boxed_accuracy_reward/std": 0.17422052025794982,
|
|
"step": 280,
|
|
"step_time": 22.47074644069944
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 443.4,
|
|
"completions/max_terminated_length": 364.0,
|
|
"completions/mean_length": 265.1375,
|
|
"completions/mean_terminated_length": 211.40833435058593,
|
|
"completions/min_length": 125.6,
|
|
"completions/min_terminated_length": 125.6,
|
|
"entropy": 0.2202783677726984,
|
|
"epoch": 0.029,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.23354178667068481,
|
|
"learning_rate": 7.1100000000000005e-06,
|
|
"loss": 0.039704719185829164,
|
|
"num_tokens": 1033446.0,
|
|
"reward": 0.2375,
|
|
"reward_std": 0.2478467583656311,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2375,
|
|
"rewards/boxed_accuracy_reward/std": 0.24784677624702453,
|
|
"step": 290,
|
|
"step_time": 29.699856723000266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 403.5,
|
|
"completions/max_terminated_length": 403.5,
|
|
"completions/mean_length": 281.5125,
|
|
"completions/mean_terminated_length": 281.5125,
|
|
"completions/min_length": 162.4,
|
|
"completions/min_terminated_length": 162.4,
|
|
"entropy": 0.2921067249029875,
|
|
"epoch": 0.03,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 7.01e-06,
|
|
"loss": 0.006478501856327057,
|
|
"num_tokens": 1071503.0,
|
|
"reward": 0.2625,
|
|
"reward_std": 0.19864802658557892,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2625,
|
|
"rewards/boxed_accuracy_reward/std": 0.19864802658557892,
|
|
"step": 300,
|
|
"step_time": 27.02418423309937
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 438.9,
|
|
"completions/max_terminated_length": 438.9,
|
|
"completions/mean_length": 247.8,
|
|
"completions/mean_terminated_length": 247.8,
|
|
"completions/min_length": 139.4,
|
|
"completions/min_terminated_length": 139.4,
|
|
"entropy": 0.22176421582698821,
|
|
"epoch": 0.031,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 1.1117596626281738,
|
|
"learning_rate": 6.91e-06,
|
|
"loss": -0.027645233273506164,
|
|
"num_tokens": 1109583.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.26302809417247774,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1625,
|
|
"rewards/boxed_accuracy_reward/std": 0.26302810609340666,
|
|
"step": 310,
|
|
"step_time": 29.716965569899912
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 388.2,
|
|
"completions/max_terminated_length": 343.9,
|
|
"completions/mean_length": 237.475,
|
|
"completions/mean_terminated_length": 228.22321472167968,
|
|
"completions/min_length": 132.2,
|
|
"completions/min_terminated_length": 132.2,
|
|
"entropy": 0.249158176779747,
|
|
"epoch": 0.032,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 6.810000000000001e-06,
|
|
"loss": -0.009709689766168594,
|
|
"num_tokens": 1142949.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.11700168251991272,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2,
|
|
"rewards/boxed_accuracy_reward/std": 0.11700168251991272,
|
|
"step": 320,
|
|
"step_time": 26.217313883200042
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 355.9,
|
|
"completions/max_terminated_length": 355.9,
|
|
"completions/mean_length": 207.9625,
|
|
"completions/mean_terminated_length": 207.9625,
|
|
"completions/min_length": 134.0,
|
|
"completions/min_terminated_length": 134.0,
|
|
"entropy": 0.257133349776268,
|
|
"epoch": 0.033,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 6.710000000000001e-06,
|
|
"loss": 0.0044826693832874295,
|
|
"num_tokens": 1177898.0,
|
|
"reward": 0.325,
|
|
"reward_std": 0.17422050833702088,
|
|
"rewards/boxed_accuracy_reward/mean": 0.325,
|
|
"rewards/boxed_accuracy_reward/std": 0.17422052025794982,
|
|
"step": 330,
|
|
"step_time": 24.09444000950098
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 455.1,
|
|
"completions/max_terminated_length": 405.7,
|
|
"completions/mean_length": 259.3375,
|
|
"completions/mean_terminated_length": 250.85535888671876,
|
|
"completions/min_length": 135.2,
|
|
"completions/min_terminated_length": 135.2,
|
|
"entropy": 0.2380140107125044,
|
|
"epoch": 0.034,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 6.610000000000001e-06,
|
|
"loss": 0.07607665061950683,
|
|
"num_tokens": 1214925.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.25040294229984283,
|
|
"rewards/boxed_accuracy_reward/mean": 0.375,
|
|
"rewards/boxed_accuracy_reward/std": 0.2504029482603073,
|
|
"step": 340,
|
|
"step_time": 30.860040666599524
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 509.9,
|
|
"completions/max_terminated_length": 390.4,
|
|
"completions/mean_length": 283.525,
|
|
"completions/mean_terminated_length": 255.67381134033204,
|
|
"completions/min_length": 167.1,
|
|
"completions/min_terminated_length": 167.1,
|
|
"entropy": 0.2493958815932274,
|
|
"epoch": 0.035,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 6.51e-06,
|
|
"loss": 0.040489372611045835,
|
|
"num_tokens": 1249775.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.15526474714279176,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1125,
|
|
"rewards/boxed_accuracy_reward/std": 0.15526476502418518,
|
|
"step": 350,
|
|
"step_time": 34.709320317299714
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 379.4,
|
|
"completions/max_terminated_length": 379.4,
|
|
"completions/mean_length": 231.2125,
|
|
"completions/mean_terminated_length": 231.2125,
|
|
"completions/min_length": 124.0,
|
|
"completions/min_terminated_length": 124.0,
|
|
"entropy": 0.268961301445961,
|
|
"epoch": 0.036,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 6.4100000000000005e-06,
|
|
"loss": -0.025786453485488893,
|
|
"num_tokens": 1284312.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.07071067690849304,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1,
|
|
"rewards/boxed_accuracy_reward/std": 0.07071067690849304,
|
|
"step": 360,
|
|
"step_time": 25.807292600200164
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 331.3,
|
|
"completions/max_terminated_length": 331.3,
|
|
"completions/mean_length": 220.925,
|
|
"completions/mean_terminated_length": 220.925,
|
|
"completions/min_length": 154.9,
|
|
"completions/min_terminated_length": 154.9,
|
|
"entropy": 0.20115781389176846,
|
|
"epoch": 0.037,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.40251433849334717,
|
|
"learning_rate": 6.3100000000000006e-06,
|
|
"loss": -0.008264368772506714,
|
|
"num_tokens": 1314754.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.10606601536273956,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1125,
|
|
"rewards/boxed_accuracy_reward/std": 0.10606601536273956,
|
|
"step": 370,
|
|
"step_time": 22.319113775398726
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 397.9,
|
|
"completions/max_terminated_length": 378.2,
|
|
"completions/mean_length": 229.525,
|
|
"completions/mean_terminated_length": 221.51964416503907,
|
|
"completions/min_length": 131.6,
|
|
"completions/min_terminated_length": 131.6,
|
|
"entropy": 0.23055790960788727,
|
|
"epoch": 0.038,
|
|
"frac_reward_zero_std": 0.9,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 6.210000000000001e-06,
|
|
"loss": 0.020930925011634828,
|
|
"num_tokens": 1347092.0,
|
|
"reward": 0.025,
|
|
"reward_std": 0.046291005611419675,
|
|
"rewards/boxed_accuracy_reward/mean": 0.025,
|
|
"rewards/boxed_accuracy_reward/std": 0.046291005611419675,
|
|
"step": 380,
|
|
"step_time": 26.844786134899188
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 486.9,
|
|
"completions/max_terminated_length": 446.0,
|
|
"completions/mean_length": 300.1,
|
|
"completions/mean_terminated_length": 291.9678588867188,
|
|
"completions/min_length": 183.8,
|
|
"completions/min_terminated_length": 183.8,
|
|
"entropy": 0.1606756892055273,
|
|
"epoch": 0.039,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 6.110000000000001e-06,
|
|
"loss": -0.025502890348434448,
|
|
"num_tokens": 1385580.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.245769664645195,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1875,
|
|
"rewards/boxed_accuracy_reward/std": 0.24576967656612397,
|
|
"step": 390,
|
|
"step_time": 32.540994760800096
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 311.1,
|
|
"completions/max_terminated_length": 311.1,
|
|
"completions/mean_length": 216.7125,
|
|
"completions/mean_terminated_length": 216.7125,
|
|
"completions/min_length": 156.9,
|
|
"completions/min_terminated_length": 156.9,
|
|
"entropy": 0.19945215657353402,
|
|
"epoch": 0.04,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.6455156803131104,
|
|
"learning_rate": 6.01e-06,
|
|
"loss": 0.026773136854171754,
|
|
"num_tokens": 1417461.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.1632926881313324,
|
|
"rewards/boxed_accuracy_reward/mean": 0.225,
|
|
"rewards/boxed_accuracy_reward/std": 0.1632926881313324,
|
|
"step": 400,
|
|
"step_time": 21.00104573589997
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 493.7,
|
|
"completions/max_terminated_length": 403.2,
|
|
"completions/mean_length": 278.675,
|
|
"completions/mean_terminated_length": 238.26178894042968,
|
|
"completions/min_length": 146.1,
|
|
"completions/min_terminated_length": 146.1,
|
|
"entropy": 0.20032316148281099,
|
|
"epoch": 0.041,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5.91e-06,
|
|
"loss": 0.045339572429656985,
|
|
"num_tokens": 1456083.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.15235702097415924,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1375,
|
|
"rewards/boxed_accuracy_reward/std": 0.15235702097415924,
|
|
"step": 410,
|
|
"step_time": 33.424314678498924
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 380.4,
|
|
"completions/max_terminated_length": 327.8,
|
|
"completions/mean_length": 234.2375,
|
|
"completions/mean_terminated_length": 225.44107360839843,
|
|
"completions/min_length": 155.1,
|
|
"completions/min_terminated_length": 155.1,
|
|
"entropy": 0.20021430365741252,
|
|
"epoch": 0.042,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5.81e-06,
|
|
"loss": 0.017741450667381288,
|
|
"num_tokens": 1487414.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.1977891743183136,
|
|
"rewards/boxed_accuracy_reward/mean": 0.3375,
|
|
"rewards/boxed_accuracy_reward/std": 0.19778918027877807,
|
|
"step": 420,
|
|
"step_time": 25.730258163498366
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 358.2,
|
|
"completions/max_terminated_length": 358.2,
|
|
"completions/mean_length": 223.7375,
|
|
"completions/mean_terminated_length": 223.7375,
|
|
"completions/min_length": 126.5,
|
|
"completions/min_terminated_length": 126.5,
|
|
"entropy": 0.27519515678286555,
|
|
"epoch": 0.043,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5.71e-06,
|
|
"loss": 0.019644300639629363,
|
|
"num_tokens": 1518057.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.19864802658557892,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1375,
|
|
"rewards/boxed_accuracy_reward/std": 0.19864802658557892,
|
|
"step": 430,
|
|
"step_time": 24.234244189500167
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 320.9,
|
|
"completions/max_terminated_length": 320.9,
|
|
"completions/mean_length": 207.3625,
|
|
"completions/mean_terminated_length": 207.3625,
|
|
"completions/min_length": 140.0,
|
|
"completions/min_terminated_length": 140.0,
|
|
"entropy": 0.23536108247935772,
|
|
"epoch": 0.044,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5.610000000000001e-06,
|
|
"loss": 0.01632075011730194,
|
|
"num_tokens": 1549726.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.13509859144687653,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2875,
|
|
"rewards/boxed_accuracy_reward/std": 0.13509859144687653,
|
|
"step": 440,
|
|
"step_time": 21.775102456000603
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 491.2,
|
|
"completions/max_terminated_length": 397.3,
|
|
"completions/mean_length": 289.6375,
|
|
"completions/mean_terminated_length": 272.23036041259763,
|
|
"completions/min_length": 176.8,
|
|
"completions/min_terminated_length": 176.8,
|
|
"entropy": 0.20252985879778862,
|
|
"epoch": 0.045,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5.510000000000001e-06,
|
|
"loss": 0.07112041711807252,
|
|
"num_tokens": 1586873.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.23691109120845794,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2,
|
|
"rewards/boxed_accuracy_reward/std": 0.23691110908985138,
|
|
"step": 450,
|
|
"step_time": 33.158058354499374
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 447.2,
|
|
"completions/max_terminated_length": 447.2,
|
|
"completions/mean_length": 284.3875,
|
|
"completions/mean_terminated_length": 284.3875,
|
|
"completions/min_length": 166.0,
|
|
"completions/min_terminated_length": 166.0,
|
|
"entropy": 0.1773521050810814,
|
|
"epoch": 0.046,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5.410000000000001e-06,
|
|
"loss": -0.04271512031555176,
|
|
"num_tokens": 1622272.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.13340125977993011,
|
|
"rewards/boxed_accuracy_reward/mean": 0.25,
|
|
"rewards/boxed_accuracy_reward/std": 0.1334012657403946,
|
|
"step": 460,
|
|
"step_time": 30.262695012098995
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 510.3,
|
|
"completions/max_terminated_length": 417.6,
|
|
"completions/mean_length": 271.6625,
|
|
"completions/mean_terminated_length": 254.18214416503906,
|
|
"completions/min_length": 141.5,
|
|
"completions/min_terminated_length": 141.5,
|
|
"entropy": 0.23274324238300323,
|
|
"epoch": 0.047,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5.310000000000001e-06,
|
|
"loss": 0.08956294059753418,
|
|
"num_tokens": 1657333.0,
|
|
"reward": 0.3,
|
|
"reward_std": 0.20411193668842315,
|
|
"rewards/boxed_accuracy_reward/mean": 0.3,
|
|
"rewards/boxed_accuracy_reward/std": 0.20411194264888763,
|
|
"step": 470,
|
|
"step_time": 34.52710425489859
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 446.0,
|
|
"completions/max_terminated_length": 396.1,
|
|
"completions/mean_length": 276.6125,
|
|
"completions/mean_terminated_length": 268.86607360839844,
|
|
"completions/min_length": 169.2,
|
|
"completions/min_terminated_length": 169.2,
|
|
"entropy": 0.25105012357234957,
|
|
"epoch": 0.048,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 5.210000000000001e-06,
|
|
"loss": -0.00809616819024086,
|
|
"num_tokens": 1694526.0,
|
|
"reward": 0.0375,
|
|
"reward_std": 0.0816463440656662,
|
|
"rewards/boxed_accuracy_reward/mean": 0.0375,
|
|
"rewards/boxed_accuracy_reward/std": 0.0816463440656662,
|
|
"step": 480,
|
|
"step_time": 30.273345093600803
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 442.4,
|
|
"completions/max_terminated_length": 381.1,
|
|
"completions/mean_length": 224.5125,
|
|
"completions/mean_terminated_length": 214.30535736083985,
|
|
"completions/min_length": 104.2,
|
|
"completions/min_terminated_length": 104.2,
|
|
"entropy": 0.24412711150944233,
|
|
"epoch": 0.049,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.4479704797267914,
|
|
"learning_rate": 5.11e-06,
|
|
"loss": 0.03438442945480347,
|
|
"num_tokens": 1726799.0,
|
|
"reward": 0.2125,
|
|
"reward_std": 0.1759178400039673,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2125,
|
|
"rewards/boxed_accuracy_reward/std": 0.17591784596443177,
|
|
"step": 490,
|
|
"step_time": 29.914187529801712
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 327.1,
|
|
"completions/max_terminated_length": 327.1,
|
|
"completions/mean_length": 217.2875,
|
|
"completions/mean_terminated_length": 217.2875,
|
|
"completions/min_length": 128.0,
|
|
"completions/min_terminated_length": 128.0,
|
|
"entropy": 0.23657882679253817,
|
|
"epoch": 0.05,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.6822343468666077,
|
|
"learning_rate": 5.01e-06,
|
|
"loss": 0.04140647947788238,
|
|
"num_tokens": 1758502.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.1632926881313324,
|
|
"rewards/boxed_accuracy_reward/mean": 0.275,
|
|
"rewards/boxed_accuracy_reward/std": 0.1632926881313324,
|
|
"step": 500,
|
|
"step_time": 22.14599506190134
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 296.2,
|
|
"completions/max_terminated_length": 296.2,
|
|
"completions/mean_length": 206.925,
|
|
"completions/mean_terminated_length": 206.925,
|
|
"completions/min_length": 134.8,
|
|
"completions/min_terminated_length": 134.8,
|
|
"entropy": 0.21120237074792386,
|
|
"epoch": 0.051,
|
|
"frac_reward_zero_std": 0.4,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.9100000000000004e-06,
|
|
"loss": -0.02729707658290863,
|
|
"num_tokens": 1787704.0,
|
|
"reward": 0.4125,
|
|
"reward_std": 0.2592615157365799,
|
|
"rewards/boxed_accuracy_reward/mean": 0.4125,
|
|
"rewards/boxed_accuracy_reward/std": 0.2592615157365799,
|
|
"step": 510,
|
|
"step_time": 20.121878111200203
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 397.7,
|
|
"completions/max_terminated_length": 397.7,
|
|
"completions/mean_length": 263.6125,
|
|
"completions/mean_terminated_length": 263.6125,
|
|
"completions/min_length": 164.6,
|
|
"completions/min_terminated_length": 164.6,
|
|
"entropy": 0.214491768181324,
|
|
"epoch": 0.052,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.8100000000000005e-06,
|
|
"loss": 0.003942453116178512,
|
|
"num_tokens": 1822977.0,
|
|
"reward": 0.0875,
|
|
"reward_std": 0.16875659823417663,
|
|
"rewards/boxed_accuracy_reward/mean": 0.0875,
|
|
"rewards/boxed_accuracy_reward/std": 0.1687566041946411,
|
|
"step": 520,
|
|
"step_time": 26.994221128601204
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0375,
|
|
"completions/max_length": 485.0,
|
|
"completions/max_terminated_length": 358.8,
|
|
"completions/mean_length": 280.6625,
|
|
"completions/mean_terminated_length": 254.4261932373047,
|
|
"completions/min_length": 165.4,
|
|
"completions/min_terminated_length": 165.4,
|
|
"entropy": 0.3250377386808395,
|
|
"epoch": 0.053,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.6389704346656799,
|
|
"learning_rate": 4.71e-06,
|
|
"loss": 0.08574633002281189,
|
|
"num_tokens": 1859638.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.15782093107700348,
|
|
"rewards/boxed_accuracy_reward/mean": 0.15,
|
|
"rewards/boxed_accuracy_reward/std": 0.15782093703746797,
|
|
"step": 530,
|
|
"step_time": 32.832222134500626
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 361.9,
|
|
"completions/max_terminated_length": 361.9,
|
|
"completions/mean_length": 252.375,
|
|
"completions/mean_terminated_length": 252.375,
|
|
"completions/min_length": 164.6,
|
|
"completions/min_terminated_length": 164.6,
|
|
"entropy": 0.2565031200647354,
|
|
"epoch": 0.054,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.610000000000001e-06,
|
|
"loss": 0.021490342915058136,
|
|
"num_tokens": 1893300.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.20871699452400208,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1625,
|
|
"rewards/boxed_accuracy_reward/std": 0.2087170124053955,
|
|
"step": 540,
|
|
"step_time": 24.475655020699925
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 373.8,
|
|
"completions/max_terminated_length": 299.6,
|
|
"completions/mean_length": 206.3,
|
|
"completions/mean_terminated_length": 185.17916717529297,
|
|
"completions/min_length": 111.3,
|
|
"completions/min_terminated_length": 111.3,
|
|
"entropy": 0.22958396673202514,
|
|
"epoch": 0.055,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.6256431937217712,
|
|
"learning_rate": 4.510000000000001e-06,
|
|
"loss": 0.03615856468677521,
|
|
"num_tokens": 1923932.0,
|
|
"reward": 0.375,
|
|
"reward_std": 0.29838343262672423,
|
|
"rewards/boxed_accuracy_reward/mean": 0.375,
|
|
"rewards/boxed_accuracy_reward/std": 0.2983834445476532,
|
|
"step": 550,
|
|
"step_time": 25.349692796901216
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 475.5,
|
|
"completions/max_terminated_length": 364.6,
|
|
"completions/mean_length": 278.4375,
|
|
"completions/mean_terminated_length": 239.52916870117187,
|
|
"completions/min_length": 158.6,
|
|
"completions/min_terminated_length": 158.6,
|
|
"entropy": 0.2459364876151085,
|
|
"epoch": 0.056,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.8983960151672363,
|
|
"learning_rate": 4.41e-06,
|
|
"loss": -0.031027239561080933,
|
|
"num_tokens": 1963111.0,
|
|
"reward": 0.075,
|
|
"reward_std": 0.18771235942840575,
|
|
"rewards/boxed_accuracy_reward/mean": 0.075,
|
|
"rewards/boxed_accuracy_reward/std": 0.18771235942840575,
|
|
"step": 560,
|
|
"step_time": 32.03998489129954
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 419.6,
|
|
"completions/max_terminated_length": 378.6,
|
|
"completions/mean_length": 257.3625,
|
|
"completions/mean_terminated_length": 249.33035888671876,
|
|
"completions/min_length": 171.2,
|
|
"completions/min_terminated_length": 171.2,
|
|
"entropy": 0.2526143785566092,
|
|
"epoch": 0.057,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.31e-06,
|
|
"loss": 0.01212334781885147,
|
|
"num_tokens": 1998908.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.14056250154972078,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2,
|
|
"rewards/boxed_accuracy_reward/std": 0.14056250751018523,
|
|
"step": 570,
|
|
"step_time": 28.352883074600687
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 394.2,
|
|
"completions/max_terminated_length": 394.2,
|
|
"completions/mean_length": 257.6875,
|
|
"completions/mean_terminated_length": 257.6875,
|
|
"completions/min_length": 160.4,
|
|
"completions/min_terminated_length": 160.4,
|
|
"entropy": 0.2496233806014061,
|
|
"epoch": 0.058,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.6807348132133484,
|
|
"learning_rate": 4.21e-06,
|
|
"loss": 0.0322126567363739,
|
|
"num_tokens": 2034387.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.1767766922712326,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1375,
|
|
"rewards/boxed_accuracy_reward/std": 0.1767766922712326,
|
|
"step": 580,
|
|
"step_time": 26.78779618450062
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 371.7,
|
|
"completions/max_terminated_length": 302.6,
|
|
"completions/mean_length": 208.2625,
|
|
"completions/mean_terminated_length": 198.36964416503906,
|
|
"completions/min_length": 121.9,
|
|
"completions/min_terminated_length": 121.9,
|
|
"entropy": 0.22029370740056037,
|
|
"epoch": 0.059,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.1100000000000005e-06,
|
|
"loss": 0.0259624183177948,
|
|
"num_tokens": 2067680.0,
|
|
"reward": 0.275,
|
|
"reward_std": 0.18685350716114044,
|
|
"rewards/boxed_accuracy_reward/mean": 0.275,
|
|
"rewards/boxed_accuracy_reward/std": 0.18685351312160492,
|
|
"step": 590,
|
|
"step_time": 25.24782887190013
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 447.7,
|
|
"completions/max_terminated_length": 380.5,
|
|
"completions/mean_length": 247.1375,
|
|
"completions/mean_terminated_length": 229.69286193847657,
|
|
"completions/min_length": 137.8,
|
|
"completions/min_terminated_length": 137.8,
|
|
"entropy": 0.24857322350144387,
|
|
"epoch": 0.06,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 4.0100000000000006e-06,
|
|
"loss": 0.04133725166320801,
|
|
"num_tokens": 2101531.0,
|
|
"reward": 0.1,
|
|
"reward_std": 0.13340125977993011,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1,
|
|
"rewards/boxed_accuracy_reward/std": 0.1334012657403946,
|
|
"step": 600,
|
|
"step_time": 30.182934339900385
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 423.4,
|
|
"completions/max_terminated_length": 362.5,
|
|
"completions/mean_length": 264.475,
|
|
"completions/mean_terminated_length": 255.80714416503906,
|
|
"completions/min_length": 171.7,
|
|
"completions/min_terminated_length": 171.7,
|
|
"entropy": 0.23762509562075138,
|
|
"epoch": 0.061,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.8978026509284973,
|
|
"learning_rate": 3.910000000000001e-06,
|
|
"loss": 0.04321916103363037,
|
|
"num_tokens": 2137113.0,
|
|
"reward": 0.4,
|
|
"reward_std": 0.24030575454235076,
|
|
"rewards/boxed_accuracy_reward/mean": 0.4,
|
|
"rewards/boxed_accuracy_reward/std": 0.24030576050281524,
|
|
"step": 610,
|
|
"step_time": 28.61728952409976
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 368.3,
|
|
"completions/max_terminated_length": 368.3,
|
|
"completions/mean_length": 232.825,
|
|
"completions/mean_terminated_length": 232.825,
|
|
"completions/min_length": 147.5,
|
|
"completions/min_terminated_length": 147.5,
|
|
"entropy": 0.2583306774497032,
|
|
"epoch": 0.062,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.4808867573738098,
|
|
"learning_rate": 3.8100000000000004e-06,
|
|
"loss": 0.0032923467457294463,
|
|
"num_tokens": 2169507.0,
|
|
"reward": 0.0625,
|
|
"reward_std": 0.12793734967708587,
|
|
"rewards/boxed_accuracy_reward/mean": 0.0625,
|
|
"rewards/boxed_accuracy_reward/std": 0.12793734967708587,
|
|
"step": 620,
|
|
"step_time": 24.657616854002118
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 393.6,
|
|
"completions/max_terminated_length": 393.6,
|
|
"completions/mean_length": 239.7875,
|
|
"completions/mean_terminated_length": 239.7875,
|
|
"completions/min_length": 150.1,
|
|
"completions/min_terminated_length": 150.1,
|
|
"entropy": 0.24895308986306192,
|
|
"epoch": 0.063,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.7100000000000005e-06,
|
|
"loss": -0.008826246112585067,
|
|
"num_tokens": 2202426.0,
|
|
"reward": 0.1625,
|
|
"reward_std": 0.19062793254852295,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1625,
|
|
"rewards/boxed_accuracy_reward/std": 0.19062793850898743,
|
|
"step": 630,
|
|
"step_time": 26.481925834499272
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 490.3,
|
|
"completions/max_terminated_length": 412.0,
|
|
"completions/mean_length": 271.6625,
|
|
"completions/mean_terminated_length": 261.0321441650391,
|
|
"completions/min_length": 156.2,
|
|
"completions/min_terminated_length": 156.2,
|
|
"entropy": 0.2555754192173481,
|
|
"epoch": 0.064,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.61e-06,
|
|
"loss": -0.015103229880332946,
|
|
"num_tokens": 2237311.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.13340125977993011,
|
|
"rewards/boxed_accuracy_reward/mean": 0.225,
|
|
"rewards/boxed_accuracy_reward/std": 0.1334012657403946,
|
|
"step": 640,
|
|
"step_time": 32.94274350120031
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 445.9,
|
|
"completions/max_terminated_length": 445.9,
|
|
"completions/mean_length": 247.9125,
|
|
"completions/mean_terminated_length": 247.9125,
|
|
"completions/min_length": 134.0,
|
|
"completions/min_terminated_length": 134.0,
|
|
"entropy": 0.28245309442281724,
|
|
"epoch": 0.065,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.5100000000000003e-06,
|
|
"loss": 0.05236095190048218,
|
|
"num_tokens": 2271440.0,
|
|
"reward": 0.15,
|
|
"reward_std": 0.18685350716114044,
|
|
"rewards/boxed_accuracy_reward/mean": 0.15,
|
|
"rewards/boxed_accuracy_reward/std": 0.18685351312160492,
|
|
"step": 650,
|
|
"step_time": 30.18701468149884
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 364.8,
|
|
"completions/max_terminated_length": 317.3,
|
|
"completions/mean_length": 223.35,
|
|
"completions/mean_terminated_length": 214.17857360839844,
|
|
"completions/min_length": 135.9,
|
|
"completions/min_terminated_length": 135.9,
|
|
"entropy": 0.265711085498333,
|
|
"epoch": 0.066,
|
|
"frac_reward_zero_std": 0.3,
|
|
"grad_norm": 0.8590646982192993,
|
|
"learning_rate": 3.4100000000000004e-06,
|
|
"loss": 0.03245477676391602,
|
|
"num_tokens": 2304332.0,
|
|
"reward": 0.3375,
|
|
"reward_std": 0.3311904340982437,
|
|
"rewards/boxed_accuracy_reward/mean": 0.3375,
|
|
"rewards/boxed_accuracy_reward/std": 0.3311904460191727,
|
|
"step": 660,
|
|
"step_time": 24.758795369901055
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 369.7,
|
|
"completions/max_terminated_length": 369.7,
|
|
"completions/mean_length": 255.3375,
|
|
"completions/mean_terminated_length": 255.3375,
|
|
"completions/min_length": 146.1,
|
|
"completions/min_terminated_length": 146.1,
|
|
"entropy": 0.2967137709259987,
|
|
"epoch": 0.067,
|
|
"frac_reward_zero_std": 0.8,
|
|
"grad_norm": 0.8069291710853577,
|
|
"learning_rate": 3.3100000000000005e-06,
|
|
"loss": 0.022205421328544618,
|
|
"num_tokens": 2338295.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.10350983142852783,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2,
|
|
"rewards/boxed_accuracy_reward/std": 0.10350984334945679,
|
|
"step": 670,
|
|
"step_time": 24.84447250970188
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 378.8,
|
|
"completions/max_terminated_length": 378.8,
|
|
"completions/mean_length": 223.475,
|
|
"completions/mean_terminated_length": 223.475,
|
|
"completions/min_length": 140.8,
|
|
"completions/min_terminated_length": 140.8,
|
|
"entropy": 0.3041005529463291,
|
|
"epoch": 0.068,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 3.21e-06,
|
|
"loss": -0.020687612891197204,
|
|
"num_tokens": 2369605.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.24954409003257752,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2,
|
|
"rewards/boxed_accuracy_reward/std": 0.24954410195350646,
|
|
"step": 680,
|
|
"step_time": 25.69439109330124
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 346.6,
|
|
"completions/max_terminated_length": 346.6,
|
|
"completions/mean_length": 237.9625,
|
|
"completions/mean_terminated_length": 237.9625,
|
|
"completions/min_length": 147.4,
|
|
"completions/min_terminated_length": 147.4,
|
|
"entropy": 0.24550328701734542,
|
|
"epoch": 0.069,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.9554150700569153,
|
|
"learning_rate": 3.1100000000000003e-06,
|
|
"loss": 0.013637937605381012,
|
|
"num_tokens": 2401898.0,
|
|
"reward": 0.1875,
|
|
"reward_std": 0.13509859144687653,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1875,
|
|
"rewards/boxed_accuracy_reward/std": 0.13509859144687653,
|
|
"step": 690,
|
|
"step_time": 23.408739431599678
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 390.2,
|
|
"completions/max_terminated_length": 390.2,
|
|
"completions/mean_length": 242.8125,
|
|
"completions/mean_terminated_length": 242.8125,
|
|
"completions/min_length": 145.6,
|
|
"completions/min_terminated_length": 145.6,
|
|
"entropy": 0.2589107438921928,
|
|
"epoch": 0.07,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 1.1996217966079712,
|
|
"learning_rate": 3.01e-06,
|
|
"loss": 0.05301226377487182,
|
|
"num_tokens": 2436907.0,
|
|
"reward": 0.1125,
|
|
"reward_std": 0.18138959705829621,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1125,
|
|
"rewards/boxed_accuracy_reward/std": 0.18138959705829621,
|
|
"step": 700,
|
|
"step_time": 26.322794716500358
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 369.9,
|
|
"completions/max_terminated_length": 291.4,
|
|
"completions/mean_length": 211.5125,
|
|
"completions/mean_terminated_length": 200.55178680419922,
|
|
"completions/min_length": 133.0,
|
|
"completions/min_terminated_length": 133.0,
|
|
"entropy": 0.19192061349749565,
|
|
"epoch": 0.071,
|
|
"frac_reward_zero_std": 0.9,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.91e-06,
|
|
"loss": -0.003464340791106224,
|
|
"num_tokens": 2467292.0,
|
|
"reward": 0.2875,
|
|
"reward_std": 0.03535533845424652,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2875,
|
|
"rewards/boxed_accuracy_reward/std": 0.03535533845424652,
|
|
"step": 710,
|
|
"step_time": 25.09874547720101
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05,
|
|
"completions/max_length": 574.9,
|
|
"completions/max_terminated_length": 388.2,
|
|
"completions/mean_length": 293.5125,
|
|
"completions/mean_terminated_length": 260.60000762939455,
|
|
"completions/min_length": 169.5,
|
|
"completions/min_terminated_length": 169.5,
|
|
"entropy": 0.22459689602255822,
|
|
"epoch": 0.072,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.2952307164669037,
|
|
"learning_rate": 2.8100000000000006e-06,
|
|
"loss": 0.04532091617584229,
|
|
"num_tokens": 2502805.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.15782093107700348,
|
|
"rewards/boxed_accuracy_reward/mean": 0.25,
|
|
"rewards/boxed_accuracy_reward/std": 0.15782093703746797,
|
|
"step": 720,
|
|
"step_time": 38.782466992000266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.9,
|
|
"completions/max_terminated_length": 315.9,
|
|
"completions/mean_length": 216.875,
|
|
"completions/mean_terminated_length": 216.875,
|
|
"completions/min_length": 130.2,
|
|
"completions/min_terminated_length": 130.2,
|
|
"entropy": 0.26642010658979415,
|
|
"epoch": 0.073,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.7213253974914551,
|
|
"learning_rate": 2.7100000000000003e-06,
|
|
"loss": -0.004600425064563751,
|
|
"num_tokens": 2535131.0,
|
|
"reward": 0.1375,
|
|
"reward_std": 0.17422835528850555,
|
|
"rewards/boxed_accuracy_reward/mean": 0.1375,
|
|
"rewards/boxed_accuracy_reward/std": 0.17422835528850555,
|
|
"step": 730,
|
|
"step_time": 21.296572963699145
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 427.0,
|
|
"completions/max_terminated_length": 427.0,
|
|
"completions/mean_length": 246.8375,
|
|
"completions/mean_terminated_length": 246.8375,
|
|
"completions/min_length": 159.4,
|
|
"completions/min_terminated_length": 159.4,
|
|
"entropy": 0.2320882935076952,
|
|
"epoch": 0.074,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.6100000000000004e-06,
|
|
"loss": -0.0016866859048604965,
|
|
"num_tokens": 2567462.0,
|
|
"reward": 0.225,
|
|
"reward_std": 0.1796922653913498,
|
|
"rewards/boxed_accuracy_reward/mean": 0.225,
|
|
"rewards/boxed_accuracy_reward/std": 0.17969227135181426,
|
|
"step": 740,
|
|
"step_time": 28.78491548419988
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0125,
|
|
"completions/max_length": 407.3,
|
|
"completions/max_terminated_length": 341.9,
|
|
"completions/mean_length": 229.175,
|
|
"completions/mean_terminated_length": 218.69285736083984,
|
|
"completions/min_length": 129.1,
|
|
"completions/min_terminated_length": 129.1,
|
|
"entropy": 0.2695804834365845,
|
|
"epoch": 0.075,
|
|
"frac_reward_zero_std": 0.6,
|
|
"grad_norm": 1.118378758430481,
|
|
"learning_rate": 2.51e-06,
|
|
"loss": 0.02082826942205429,
|
|
"num_tokens": 2601372.0,
|
|
"reward": 0.175,
|
|
"reward_std": 0.17045392990112304,
|
|
"rewards/boxed_accuracy_reward/mean": 0.175,
|
|
"rewards/boxed_accuracy_reward/std": 0.17045392990112304,
|
|
"step": 750,
|
|
"step_time": 27.563064585998653
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 250.8,
|
|
"completions/max_terminated_length": 250.8,
|
|
"completions/mean_length": 172.0375,
|
|
"completions/mean_terminated_length": 172.0375,
|
|
"completions/min_length": 121.6,
|
|
"completions/min_terminated_length": 121.6,
|
|
"entropy": 0.2495213456451893,
|
|
"epoch": 0.076,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 1.1198997497558594,
|
|
"learning_rate": 2.4100000000000002e-06,
|
|
"loss": -0.0004969652742147446,
|
|
"num_tokens": 2631303.0,
|
|
"reward": 0.3125,
|
|
"reward_std": 0.14433692693710326,
|
|
"rewards/boxed_accuracy_reward/mean": 0.3125,
|
|
"rewards/boxed_accuracy_reward/std": 0.14433693289756774,
|
|
"step": 760,
|
|
"step_time": 17.046090553899557
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 341.0,
|
|
"completions/max_terminated_length": 341.0,
|
|
"completions/mean_length": 209.725,
|
|
"completions/mean_terminated_length": 209.725,
|
|
"completions/min_length": 120.9,
|
|
"completions/min_terminated_length": 120.9,
|
|
"entropy": 0.2746701970696449,
|
|
"epoch": 0.077,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.3100000000000003e-06,
|
|
"loss": 0.04669084250926971,
|
|
"num_tokens": 2661489.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.23691109120845794,
|
|
"rewards/boxed_accuracy_reward/mean": 0.25,
|
|
"rewards/boxed_accuracy_reward/std": 0.23691110908985138,
|
|
"step": 770,
|
|
"step_time": 23.079953304801894
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0625,
|
|
"completions/max_length": 599.1,
|
|
"completions/max_terminated_length": 445.5,
|
|
"completions/mean_length": 323.7625,
|
|
"completions/mean_terminated_length": 287.5857177734375,
|
|
"completions/min_length": 165.4,
|
|
"completions/min_terminated_length": 165.4,
|
|
"entropy": 0.20161464884877206,
|
|
"epoch": 0.078,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.6654395461082458,
|
|
"learning_rate": 2.21e-06,
|
|
"loss": -0.01412966102361679,
|
|
"num_tokens": 2703534.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.14056250154972078,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2,
|
|
"rewards/boxed_accuracy_reward/std": 0.14056250751018523,
|
|
"step": 780,
|
|
"step_time": 40.28982557420022
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025,
|
|
"completions/max_length": 488.7,
|
|
"completions/max_terminated_length": 422.1,
|
|
"completions/mean_length": 281.3625,
|
|
"completions/mean_terminated_length": 261.6958343505859,
|
|
"completions/min_length": 144.9,
|
|
"completions/min_terminated_length": 144.9,
|
|
"entropy": 0.23346679359674455,
|
|
"epoch": 0.079,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.11e-06,
|
|
"loss": 0.03522632122039795,
|
|
"num_tokens": 2739035.0,
|
|
"reward": 0.25,
|
|
"reward_std": 0.2529387533664703,
|
|
"rewards/boxed_accuracy_reward/mean": 0.25,
|
|
"rewards/boxed_accuracy_reward/std": 0.2529387533664703,
|
|
"step": 790,
|
|
"step_time": 33.08498501170034
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 344.0,
|
|
"completions/max_terminated_length": 344.0,
|
|
"completions/mean_length": 210.85,
|
|
"completions/mean_terminated_length": 210.85,
|
|
"completions/min_length": 133.1,
|
|
"completions/min_terminated_length": 133.1,
|
|
"entropy": 0.25274474173784256,
|
|
"epoch": 0.08,
|
|
"frac_reward_zero_std": 0.7,
|
|
"grad_norm": 0.33814314007759094,
|
|
"learning_rate": 2.0100000000000002e-06,
|
|
"loss": 0.009773983806371688,
|
|
"num_tokens": 2768951.0,
|
|
"reward": 0.2,
|
|
"reward_std": 0.13340125977993011,
|
|
"rewards/boxed_accuracy_reward/mean": 0.2,
|
|
"rewards/boxed_accuracy_reward/std": 0.1334012657403946,
|
|
"step": 800,
|
|
"step_time": 23.216696536900418
|
|
}
|
|
],
|
|
"logging_steps": 10,
|
|
"max_steps": 1000,
|
|
"num_input_tokens_seen": 2768951,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 200,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|