4835 lines
202 KiB
JSON
4835 lines
202 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 2.0,
|
|
"eval_steps": 10,
|
|
"global_step": 136,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"epoch": 0,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.00992063521629288,
|
|
"eval_completions/max_length": 1050.8809523809523,
|
|
"eval_completions/max_terminated_length": 925.8571428571429,
|
|
"eval_completions/mean_length": 556.7480287097749,
|
|
"eval_completions/mean_terminated_length": 531.9472343808129,
|
|
"eval_completions/min_length": 248.51190476190476,
|
|
"eval_completions/min_terminated_length": 248.51190476190476,
|
|
"eval_entropy": 0.2654567967568125,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 0.0,
|
|
"eval_reward": 0.6587301735721883,
|
|
"eval_reward_std": 0.43563264750299,
|
|
"eval_rewards/reward_correctness/mean": 0.6587301629285017,
|
|
"eval_rewards/reward_correctness/std": 0.4356326439550945,
|
|
"eval_runtime": 645.7404,
|
|
"eval_samples_per_second": 0.774,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.10470169515777646,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.01892138532918845,
|
|
"eval_sampling/importance_sampling_ratio/min": 7.668724304032257e-06,
|
|
"eval_sampling/sampling_logp_difference/max": 2.6867869666644504,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.06213199950399853,
|
|
"eval_steps_per_second": 0.13,
|
|
"step": 0
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0240885429084301,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2792.0,
|
|
"completions/mean_length": 649.4779052734375,
|
|
"completions/mean_terminated_length": 589.7718505859375,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 0.5836714202305302,
|
|
"epoch": 0.014705882352941176,
|
|
"frac_reward_zero_std": 0.4765625,
|
|
"grad_norm": 0.0848995053910046,
|
|
"learning_rate": 0.0,
|
|
"loss": -0.0115,
|
|
"num_tokens": 1152974.0,
|
|
"reward": 0.4778645932674408,
|
|
"reward_std": 0.49967244267463684,
|
|
"rewards/reward_correctness/mean": 0.4778645932674408,
|
|
"rewards/reward_correctness/std": 0.49967244267463684,
|
|
"sampling/importance_sampling_ratio/max": 2.935307502746582,
|
|
"sampling/importance_sampling_ratio/mean": 0.42810118198394775,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 13.99999713897705,
|
|
"sampling/sampling_logp_difference/mean": 0.03527523949742317,
|
|
"step": 1,
|
|
"step_time": 313.13801932055503
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.02278645895421505,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2986.0,
|
|
"completions/mean_length": 633.5443115234375,
|
|
"completions/mean_terminated_length": 576.7142333984375,
|
|
"completions/min_length": 88.0,
|
|
"completions/min_terminated_length": 88.0,
|
|
"entropy": 0.578207426937297,
|
|
"epoch": 0.029411764705882353,
|
|
"frac_reward_zero_std": 0.40625,
|
|
"grad_norm": 0.06942535958988857,
|
|
"learning_rate": 6.000000000000001e-07,
|
|
"loss": 0.0013,
|
|
"num_tokens": 2271322.0,
|
|
"reward": 0.4720052182674408,
|
|
"reward_std": 0.4993782341480255,
|
|
"rewards/reward_correctness/mean": 0.4720052182674408,
|
|
"rewards/reward_correctness/std": 0.4993782639503479,
|
|
"sampling/importance_sampling_ratio/max": 2.9959943294525146,
|
|
"sampling/importance_sampling_ratio/mean": 0.5243185758590698,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.144627571105957,
|
|
"sampling/sampling_logp_difference/mean": 0.023212533444166183,
|
|
"step": 2,
|
|
"step_time": 272.8161257347092
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.02473958395421505,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2565.0,
|
|
"completions/mean_length": 632.01953125,
|
|
"completions/mean_terminated_length": 570.1242065429688,
|
|
"completions/min_length": 32.0,
|
|
"completions/min_terminated_length": 32.0,
|
|
"entropy": 0.5463347954209894,
|
|
"epoch": 0.04411764705882353,
|
|
"frac_reward_zero_std": 0.3515625,
|
|
"grad_norm": 0.09186588323837147,
|
|
"learning_rate": 1.2000000000000002e-06,
|
|
"loss": -0.0343,
|
|
"num_tokens": 3370264.0,
|
|
"reward": 0.5319010615348816,
|
|
"reward_std": 0.4991437494754791,
|
|
"rewards/reward_correctness/mean": 0.5319010615348816,
|
|
"rewards/reward_correctness/std": 0.4991438090801239,
|
|
"sampling/importance_sampling_ratio/max": 2.989497184753418,
|
|
"sampling/importance_sampling_ratio/mean": 0.5294168591499329,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9646859169006348,
|
|
"sampling/sampling_logp_difference/mean": 0.02268599346280098,
|
|
"step": 3,
|
|
"step_time": 267.1632728436962
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.02669270895421505,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2661.0,
|
|
"completions/mean_length": 649.9661865234375,
|
|
"completions/mean_terminated_length": 583.54248046875,
|
|
"completions/min_length": 7.0,
|
|
"completions/min_terminated_length": 7.0,
|
|
"entropy": 0.6193987773731351,
|
|
"epoch": 0.058823529411764705,
|
|
"frac_reward_zero_std": 0.34375,
|
|
"grad_norm": 0.12364894599624972,
|
|
"learning_rate": 1.8e-06,
|
|
"loss": -0.0568,
|
|
"num_tokens": 4516584.0,
|
|
"reward": 0.423828125,
|
|
"reward_std": 0.4943247139453888,
|
|
"rewards/reward_correctness/mean": 0.423828125,
|
|
"rewards/reward_correctness/std": 0.4943247139453888,
|
|
"sampling/importance_sampling_ratio/max": 2.9678268432617188,
|
|
"sampling/importance_sampling_ratio/mean": 0.5203623175621033,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9797148704528809,
|
|
"sampling/sampling_logp_difference/mean": 0.02456444501876831,
|
|
"step": 4,
|
|
"step_time": 272.84204353392124
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0240885429084301,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2579.0,
|
|
"completions/mean_length": 646.568359375,
|
|
"completions/mean_terminated_length": 586.701171875,
|
|
"completions/min_length": 51.0,
|
|
"completions/min_terminated_length": 51.0,
|
|
"entropy": 0.6028780657798052,
|
|
"epoch": 0.07352941176470588,
|
|
"frac_reward_zero_std": 0.421875,
|
|
"grad_norm": 0.08114140165877926,
|
|
"learning_rate": 2.4000000000000003e-06,
|
|
"loss": -0.0142,
|
|
"num_tokens": 5663145.0,
|
|
"reward": 0.4537760615348816,
|
|
"reward_std": 0.49802088737487793,
|
|
"rewards/reward_correctness/mean": 0.4537760317325592,
|
|
"rewards/reward_correctness/std": 0.4980209469795227,
|
|
"sampling/importance_sampling_ratio/max": 2.9854531288146973,
|
|
"sampling/importance_sampling_ratio/mean": 0.5120000839233398,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.7712206840515137,
|
|
"sampling/sampling_logp_difference/mean": 0.023626327514648438,
|
|
"step": 5,
|
|
"step_time": 278.0259652659297
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.02669270895421505,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2117.0,
|
|
"completions/mean_length": 635.78125,
|
|
"completions/mean_terminated_length": 568.9685668945312,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 0.5801784613868222,
|
|
"epoch": 0.08823529411764706,
|
|
"frac_reward_zero_std": 0.484375,
|
|
"grad_norm": 0.05935708343875522,
|
|
"learning_rate": 3e-06,
|
|
"loss": -0.0204,
|
|
"num_tokens": 6771561.0,
|
|
"reward": 0.5423177480697632,
|
|
"reward_std": 0.49836820363998413,
|
|
"rewards/reward_correctness/mean": 0.5423176884651184,
|
|
"rewards/reward_correctness/std": 0.4983682334423065,
|
|
"sampling/importance_sampling_ratio/max": 2.988870859146118,
|
|
"sampling/importance_sampling_ratio/mean": 0.5353550314903259,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 2.0358147621154785,
|
|
"sampling/sampling_logp_difference/mean": 0.023153454065322876,
|
|
"step": 6,
|
|
"step_time": 274.2749971947633
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0325520858168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1737.0,
|
|
"completions/mean_length": 651.171875,
|
|
"completions/mean_terminated_length": 569.7173461914062,
|
|
"completions/min_length": 106.0,
|
|
"completions/min_terminated_length": 106.0,
|
|
"entropy": 0.570356258423999,
|
|
"epoch": 0.10294117647058823,
|
|
"frac_reward_zero_std": 0.546875,
|
|
"grad_norm": 0.06858649907867036,
|
|
"learning_rate": 2.9996118137817615e-06,
|
|
"loss": -0.0121,
|
|
"num_tokens": 7907301.0,
|
|
"reward": 0.55078125,
|
|
"reward_std": 0.4975765645503998,
|
|
"rewards/reward_correctness/mean": 0.55078125,
|
|
"rewards/reward_correctness/std": 0.4975765645503998,
|
|
"sampling/importance_sampling_ratio/max": 2.9824862480163574,
|
|
"sampling/importance_sampling_ratio/mean": 0.5241525173187256,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.0719003677368164,
|
|
"sampling/sampling_logp_difference/mean": 0.023694580420851707,
|
|
"step": 7,
|
|
"step_time": 274.51085175946355
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0338541679084301,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2899.0,
|
|
"completions/mean_length": 716.3203125,
|
|
"completions/mean_terminated_length": 633.7762451171875,
|
|
"completions/min_length": 166.0,
|
|
"completions/min_terminated_length": 166.0,
|
|
"entropy": 0.5644197696819901,
|
|
"epoch": 0.11764705882352941,
|
|
"frac_reward_zero_std": 0.453125,
|
|
"grad_norm": 0.06117333934464337,
|
|
"learning_rate": 2.998447478369329e-06,
|
|
"loss": -0.0272,
|
|
"num_tokens": 9139437.0,
|
|
"reward": 0.4895833432674408,
|
|
"reward_std": 0.5000542402267456,
|
|
"rewards/reward_correctness/mean": 0.4895833432674408,
|
|
"rewards/reward_correctness/std": 0.5000542402267456,
|
|
"sampling/importance_sampling_ratio/max": 2.9775948524475098,
|
|
"sampling/importance_sampling_ratio/mean": 0.5255301594734192,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.0037541389465332,
|
|
"sampling/sampling_logp_difference/mean": 0.021436309441924095,
|
|
"step": 8,
|
|
"step_time": 269.14979780837893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.021484375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2486.0,
|
|
"completions/mean_length": 656.640625,
|
|
"completions/mean_terminated_length": 603.6087646484375,
|
|
"completions/min_length": 109.0,
|
|
"completions/min_terminated_length": 109.0,
|
|
"entropy": 0.5387037470936775,
|
|
"epoch": 0.1323529411764706,
|
|
"frac_reward_zero_std": 0.53125,
|
|
"grad_norm": 0.05537902992630806,
|
|
"learning_rate": 2.9965076633611604e-06,
|
|
"loss": -0.002,
|
|
"num_tokens": 10282293.0,
|
|
"reward": 0.572265625,
|
|
"reward_std": 0.49491122364997864,
|
|
"rewards/reward_correctness/mean": 0.572265625,
|
|
"rewards/reward_correctness/std": 0.494911253452301,
|
|
"sampling/importance_sampling_ratio/max": 2.991780996322632,
|
|
"sampling/importance_sampling_ratio/mean": 0.5056574940681458,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.994542121887207,
|
|
"sampling/sampling_logp_difference/mean": 0.02303769439458847,
|
|
"step": 9,
|
|
"step_time": 277.4853810737841
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.02083333395421505,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 3008.0,
|
|
"completions/mean_length": 643.1126708984375,
|
|
"completions/mean_terminated_length": 591.442138671875,
|
|
"completions/min_length": 155.0,
|
|
"completions/min_terminated_length": 155.0,
|
|
"entropy": 0.47972284245770425,
|
|
"epoch": 0.14705882352941177,
|
|
"frac_reward_zero_std": 0.4765625,
|
|
"grad_norm": 0.05498481378949289,
|
|
"learning_rate": 2.993793484326816e-06,
|
|
"loss": -0.0034,
|
|
"num_tokens": 11421770.0,
|
|
"reward": 0.5618489980697632,
|
|
"reward_std": 0.49632149934768677,
|
|
"rewards/reward_correctness/mean": 0.5618489384651184,
|
|
"rewards/reward_correctness/std": 0.49632152915000916,
|
|
"sampling/importance_sampling_ratio/max": 2.9866268634796143,
|
|
"sampling/importance_sampling_ratio/mean": 0.5399176478385925,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9893784523010254,
|
|
"sampling/sampling_logp_difference/mean": 0.0213819220662117,
|
|
"step": 10,
|
|
"step_time": 276.4947049263865
|
|
},
|
|
{
|
|
"epoch": 0.14705882352941177,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.007936508173034304,
|
|
"eval_completions/max_length": 1043.8095238095239,
|
|
"eval_completions/max_terminated_length": 943.4166666666666,
|
|
"eval_completions/mean_length": 576.2758113316128,
|
|
"eval_completions/mean_terminated_length": 556.6127137683686,
|
|
"eval_completions/min_length": 281.51190476190476,
|
|
"eval_completions/min_terminated_length": 281.51190476190476,
|
|
"eval_entropy": 0.2537455828416915,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 11421770.0,
|
|
"eval_reward": 0.6884920799306461,
|
|
"eval_reward_std": 0.42304758372760953,
|
|
"eval_rewards/reward_correctness/mean": 0.6884920650294849,
|
|
"eval_rewards/reward_correctness/std": 0.4230475830180304,
|
|
"eval_runtime": 638.6604,
|
|
"eval_samples_per_second": 0.783,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.19307976385317444,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.03580059619768522,
|
|
"eval_sampling/importance_sampling_ratio/min": 3.407798286771119e-06,
|
|
"eval_sampling/sampling_logp_difference/max": 2.7100065719513666,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.059989471831137224,
|
|
"eval_steps_per_second": 0.132,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.02473958395421505,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2715.0,
|
|
"completions/mean_length": 639.107421875,
|
|
"completions/mean_terminated_length": 577.3919067382812,
|
|
"completions/min_length": 136.0,
|
|
"completions/min_terminated_length": 136.0,
|
|
"entropy": 0.5096990133170038,
|
|
"epoch": 0.16176470588235295,
|
|
"frac_reward_zero_std": 0.421875,
|
|
"grad_norm": 0.057721741043468014,
|
|
"learning_rate": 2.990306502165398e-06,
|
|
"loss": -0.0187,
|
|
"num_tokens": 12545459.0,
|
|
"reward": 0.57421875,
|
|
"reward_std": 0.4946219027042389,
|
|
"rewards/reward_correctness/mean": 0.57421875,
|
|
"rewards/reward_correctness/std": 0.4946219325065613,
|
|
"sampling/importance_sampling_ratio/max": 2.979454755783081,
|
|
"sampling/importance_sampling_ratio/mean": 0.5547703504562378,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9523108005523682,
|
|
"sampling/sampling_logp_difference/mean": 0.02195746824145317,
|
|
"step": 11,
|
|
"step_time": 258.60902352258563
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0299479179084301,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2413.0,
|
|
"completions/mean_length": 672.0462646484375,
|
|
"completions/mean_terminated_length": 597.9537353515625,
|
|
"completions/min_length": 133.0,
|
|
"completions/min_terminated_length": 133.0,
|
|
"entropy": 0.5439460552297533,
|
|
"epoch": 0.17647058823529413,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.05418244520480129,
|
|
"learning_rate": 2.986048722207899e-06,
|
|
"loss": -0.0058,
|
|
"num_tokens": 13729294.0,
|
|
"reward": 0.5579427480697632,
|
|
"reward_std": 0.4967930018901825,
|
|
"rewards/reward_correctness/mean": 0.5579426884651184,
|
|
"rewards/reward_correctness/std": 0.4967930316925049,
|
|
"sampling/importance_sampling_ratio/max": 2.9794344902038574,
|
|
"sampling/importance_sampling_ratio/mean": 0.5251661539077759,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.4261195659637451,
|
|
"sampling/sampling_logp_difference/mean": 0.02223217487335205,
|
|
"step": 12,
|
|
"step_time": 267.7675925884396
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.025390625,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2197.0,
|
|
"completions/mean_length": 665.5872802734375,
|
|
"completions/mean_terminated_length": 603.4348754882812,
|
|
"completions/min_length": 139.0,
|
|
"completions/min_terminated_length": 139.0,
|
|
"entropy": 0.5305114695802331,
|
|
"epoch": 0.19117647058823528,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.05705325757219269,
|
|
"learning_rate": 2.981022593063946e-06,
|
|
"loss": 0.0052,
|
|
"num_tokens": 14916792.0,
|
|
"reward": 0.53515625,
|
|
"reward_std": 0.4989249110221863,
|
|
"rewards/reward_correctness/mean": 0.53515625,
|
|
"rewards/reward_correctness/std": 0.49892494082450867,
|
|
"sampling/importance_sampling_ratio/max": 2.997776746749878,
|
|
"sampling/importance_sampling_ratio/mean": 0.5438298583030701,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 2.7724132537841797,
|
|
"sampling/sampling_logp_difference/mean": 0.02207307144999504,
|
|
"step": 13,
|
|
"step_time": 298.6625449950807
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0279947929084301,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2265.0,
|
|
"completions/mean_length": 690.9935302734375,
|
|
"completions/mean_terminated_length": 622.41796875,
|
|
"completions/min_length": 166.0,
|
|
"completions/min_terminated_length": 166.0,
|
|
"entropy": 0.49725982279051095,
|
|
"epoch": 0.20588235294117646,
|
|
"frac_reward_zero_std": 0.4375,
|
|
"grad_norm": 0.058814816392006475,
|
|
"learning_rate": 2.9752310052136353e-06,
|
|
"loss": 0.0141,
|
|
"num_tokens": 16113926.0,
|
|
"reward": 0.5716146230697632,
|
|
"reward_std": 0.49500590562820435,
|
|
"rewards/reward_correctness/mean": 0.5716145634651184,
|
|
"rewards/reward_correctness/std": 0.4950059652328491,
|
|
"sampling/importance_sampling_ratio/max": 2.9824302196502686,
|
|
"sampling/importance_sampling_ratio/mean": 0.5442800521850586,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.378803014755249,
|
|
"sampling/sampling_logp_difference/mean": 0.02060725726187229,
|
|
"step": 14,
|
|
"step_time": 274.99928429629654
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.029296875,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1901.0,
|
|
"completions/mean_length": 652.2025146484375,
|
|
"completions/mean_terminated_length": 579.4735107421875,
|
|
"completions/min_length": 161.0,
|
|
"completions/min_terminated_length": 161.0,
|
|
"entropy": 0.5102907492546365,
|
|
"epoch": 0.22058823529411764,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.05525897475793538,
|
|
"learning_rate": 2.968677289345242e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 17267821.0,
|
|
"reward": 0.5826823115348816,
|
|
"reward_std": 0.49327683448791504,
|
|
"rewards/reward_correctness/mean": 0.5826823115348816,
|
|
"rewards/reward_correctness/std": 0.49327683448791504,
|
|
"sampling/importance_sampling_ratio/max": 2.9618661403656006,
|
|
"sampling/importance_sampling_ratio/mean": 0.5603505969047546,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.0892324447631836,
|
|
"sampling/sampling_logp_difference/mean": 0.02108415775001049,
|
|
"step": 15,
|
|
"step_time": 297.76528844004497
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0299479179084301,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2084.0,
|
|
"completions/mean_length": 692.1009521484375,
|
|
"completions/mean_terminated_length": 618.6275634765625,
|
|
"completions/min_length": 113.0,
|
|
"completions/min_terminated_length": 113.0,
|
|
"entropy": 0.5383049458032474,
|
|
"epoch": 0.23529411764705882,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.05610150359953727,
|
|
"learning_rate": 2.9613652144397706e-06,
|
|
"loss": -0.0028,
|
|
"num_tokens": 18491496.0,
|
|
"reward": 0.5182291865348816,
|
|
"reward_std": 0.49983030557632446,
|
|
"rewards/reward_correctness/mean": 0.5182291865348816,
|
|
"rewards/reward_correctness/std": 0.49983033537864685,
|
|
"sampling/importance_sampling_ratio/max": 2.9970788955688477,
|
|
"sampling/importance_sampling_ratio/mean": 0.5334643721580505,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.979703426361084,
|
|
"sampling/sampling_logp_difference/mean": 0.02187207341194153,
|
|
"step": 16,
|
|
"step_time": 284.81178701808676
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0338541679084301,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2957.0,
|
|
"completions/mean_length": 712.1243896484375,
|
|
"completions/mean_terminated_length": 629.4332885742188,
|
|
"completions/min_length": 153.0,
|
|
"completions/min_terminated_length": 153.0,
|
|
"entropy": 0.5025974611053243,
|
|
"epoch": 0.25,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.04720701387579848,
|
|
"learning_rate": 2.9532989856034515e-06,
|
|
"loss": -0.0051,
|
|
"num_tokens": 19722251.0,
|
|
"reward": 0.556640625,
|
|
"reward_std": 0.49694323539733887,
|
|
"rewards/reward_correctness/mean": 0.556640625,
|
|
"rewards/reward_correctness/std": 0.49694326519966125,
|
|
"sampling/importance_sampling_ratio/max": 2.994859457015991,
|
|
"sampling/importance_sampling_ratio/mean": 0.5143538117408752,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9831252098083496,
|
|
"sampling/sampling_logp_difference/mean": 0.022461896762251854,
|
|
"step": 17,
|
|
"step_time": 291.982781663537
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.021484375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2772.0,
|
|
"completions/mean_length": 651.921875,
|
|
"completions/mean_terminated_length": 598.7864379882812,
|
|
"completions/min_length": 120.0,
|
|
"completions/min_terminated_length": 120.0,
|
|
"entropy": 0.4585979155963287,
|
|
"epoch": 0.2647058823529412,
|
|
"frac_reward_zero_std": 0.421875,
|
|
"grad_norm": 0.05438412892841907,
|
|
"learning_rate": 2.94448324164942e-06,
|
|
"loss": -0.0191,
|
|
"num_tokens": 20875235.0,
|
|
"reward": 0.580078125,
|
|
"reward_std": 0.49370652437210083,
|
|
"rewards/reward_correctness/mean": 0.580078125,
|
|
"rewards/reward_correctness/std": 0.4937065839767456,
|
|
"sampling/importance_sampling_ratio/max": 2.980726718902588,
|
|
"sampling/importance_sampling_ratio/mean": 0.5411784052848816,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 2.025559663772583,
|
|
"sampling/sampling_logp_difference/mean": 0.019891459494829178,
|
|
"step": 18,
|
|
"step_time": 282.89908104995266
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0416666679084301,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2449.0,
|
|
"completions/mean_length": 699.4583740234375,
|
|
"completions/mean_terminated_length": 596.3043823242188,
|
|
"completions/min_length": 172.0,
|
|
"completions/min_terminated_length": 172.0,
|
|
"entropy": 0.5197840981418267,
|
|
"epoch": 0.27941176470588236,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.04706840896526157,
|
|
"learning_rate": 2.934923052429984e-06,
|
|
"loss": -0.0131,
|
|
"num_tokens": 22092475.0,
|
|
"reward": 0.59765625,
|
|
"reward_std": 0.49053022265434265,
|
|
"rewards/reward_correctness/mean": 0.59765625,
|
|
"rewards/reward_correctness/std": 0.49053022265434265,
|
|
"sampling/importance_sampling_ratio/max": 2.9970383644104004,
|
|
"sampling/importance_sampling_ratio/mean": 0.5492980480194092,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.5002199411392212,
|
|
"sampling/sampling_logp_difference/mean": 0.02276076376438141,
|
|
"step": 19,
|
|
"step_time": 283.09366952162236
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.041015625,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1904.0,
|
|
"completions/mean_length": 695.2103271484375,
|
|
"completions/mean_terminated_length": 593.5552978515625,
|
|
"completions/min_length": 164.0,
|
|
"completions/min_terminated_length": 164.0,
|
|
"entropy": 0.5042589120566845,
|
|
"epoch": 0.29411764705882354,
|
|
"frac_reward_zero_std": 0.4140625,
|
|
"grad_norm": 0.05405444364791074,
|
|
"learning_rate": 2.924623915920992e-06,
|
|
"loss": 0.0048,
|
|
"num_tokens": 23292318.0,
|
|
"reward": 0.5670573115348816,
|
|
"reward_std": 0.4956442713737488,
|
|
"rewards/reward_correctness/mean": 0.5670573115348816,
|
|
"rewards/reward_correctness/std": 0.4956442713737488,
|
|
"sampling/importance_sampling_ratio/max": 2.979520320892334,
|
|
"sampling/importance_sampling_ratio/mean": 0.5073182582855225,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.4154164791107178,
|
|
"sampling/sampling_logp_difference/mean": 0.023046687245368958,
|
|
"step": 20,
|
|
"step_time": 277.07896974822506
|
|
},
|
|
{
|
|
"epoch": 0.29411764705882354,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.003968254086517152,
|
|
"eval_completions/max_length": 973.297619047619,
|
|
"eval_completions/max_terminated_length": 935.7857142857143,
|
|
"eval_completions/mean_length": 563.2658905755906,
|
|
"eval_completions/mean_terminated_length": 554.1896990821475,
|
|
"eval_completions/min_length": 285.7857142857143,
|
|
"eval_completions/min_terminated_length": 285.7857142857143,
|
|
"eval_entropy": 0.24252484188902945,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 23292318.0,
|
|
"eval_reward": 0.6884920824141729,
|
|
"eval_reward_std": 0.44174214807294665,
|
|
"eval_rewards/reward_correctness/mean": 0.6884920646746954,
|
|
"eval_rewards/reward_correctness/std": 0.4417421498468944,
|
|
"eval_runtime": 593.7444,
|
|
"eval_samples_per_second": 0.842,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.21161528248709752,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.04097350465803001,
|
|
"eval_sampling/importance_sampling_ratio/min": 8.914577132557873e-06,
|
|
"eval_sampling/sampling_logp_difference/max": 2.7017440568833124,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.05756539217240754,
|
|
"eval_steps_per_second": 0.141,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0436197929084301,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1927.0,
|
|
"completions/mean_length": 745.3932495117188,
|
|
"completions/mean_terminated_length": 639.3792114257812,
|
|
"completions/min_length": 168.0,
|
|
"completions/min_terminated_length": 168.0,
|
|
"entropy": 0.5028883977793157,
|
|
"epoch": 0.3088235294117647,
|
|
"frac_reward_zero_std": 0.515625,
|
|
"grad_norm": 0.03804389825609189,
|
|
"learning_rate": 2.913591755060004e-06,
|
|
"loss": -0.0281,
|
|
"num_tokens": 24588862.0,
|
|
"reward": 0.544921875,
|
|
"reward_std": 0.4981400966644287,
|
|
"rewards/reward_correctness/mean": 0.544921875,
|
|
"rewards/reward_correctness/std": 0.4981401264667511,
|
|
"sampling/importance_sampling_ratio/max": 2.961940288543701,
|
|
"sampling/importance_sampling_ratio/mean": 0.5209087133407593,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.042482852935791,
|
|
"sampling/sampling_logp_difference/mean": 0.02183881774544716,
|
|
"step": 21,
|
|
"step_time": 289.08046070858836
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0423177108168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2720.0,
|
|
"completions/mean_length": 757.6243896484375,
|
|
"completions/mean_terminated_length": 655.3576049804688,
|
|
"completions/min_length": 177.0,
|
|
"completions/min_terminated_length": 177.0,
|
|
"entropy": 0.5200647414894775,
|
|
"epoch": 0.3235294117647059,
|
|
"frac_reward_zero_std": 0.421875,
|
|
"grad_norm": 0.050298358349411876,
|
|
"learning_rate": 2.901832914340062e-06,
|
|
"loss": -0.0177,
|
|
"num_tokens": 25890189.0,
|
|
"reward": 0.53125,
|
|
"reward_std": 0.4991849958896637,
|
|
"rewards/reward_correctness/mean": 0.53125,
|
|
"rewards/reward_correctness/std": 0.4991849958896637,
|
|
"sampling/importance_sampling_ratio/max": 2.954637289047241,
|
|
"sampling/importance_sampling_ratio/mean": 0.5123823881149292,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.7315807342529297,
|
|
"sampling/sampling_logp_difference/mean": 0.022565271705389023,
|
|
"step": 22,
|
|
"step_time": 280.0331052900292
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0423177108168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2145.0,
|
|
"completions/mean_length": 717.806640625,
|
|
"completions/mean_terminated_length": 614.0183715820312,
|
|
"completions/min_length": 198.0,
|
|
"completions/min_terminated_length": 198.0,
|
|
"entropy": 0.472823026124388,
|
|
"epoch": 0.3382352941176471,
|
|
"frac_reward_zero_std": 0.515625,
|
|
"grad_norm": 0.053729117053239864,
|
|
"learning_rate": 2.889354156161033e-06,
|
|
"loss": -0.0047,
|
|
"num_tokens": 27140520.0,
|
|
"reward": 0.591796875,
|
|
"reward_std": 0.4916611313819885,
|
|
"rewards/reward_correctness/mean": 0.591796875,
|
|
"rewards/reward_correctness/std": 0.4916611909866333,
|
|
"sampling/importance_sampling_ratio/max": 2.9909863471984863,
|
|
"sampling/importance_sampling_ratio/mean": 0.5710696578025818,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.006546974182129,
|
|
"sampling/sampling_logp_difference/mean": 0.020792663097381592,
|
|
"step": 23,
|
|
"step_time": 293.80172044876963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0494791679084301,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2233.0,
|
|
"completions/mean_length": 702.1041870117188,
|
|
"completions/mean_terminated_length": 578.73974609375,
|
|
"completions/min_length": 176.0,
|
|
"completions/min_terminated_length": 176.0,
|
|
"entropy": 0.5469449365045875,
|
|
"epoch": 0.35294117647058826,
|
|
"frac_reward_zero_std": 0.5078125,
|
|
"grad_norm": 0.047428014800435944,
|
|
"learning_rate": 2.876162656940614e-06,
|
|
"loss": 0.0024,
|
|
"num_tokens": 28369072.0,
|
|
"reward": 0.5735677480697632,
|
|
"reward_std": 0.4947192072868347,
|
|
"rewards/reward_correctness/mean": 0.5735676884651184,
|
|
"rewards/reward_correctness/std": 0.4947192370891571,
|
|
"sampling/importance_sampling_ratio/max": 2.939020872116089,
|
|
"sampling/importance_sampling_ratio/mean": 0.5351980328559875,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9955153465270996,
|
|
"sampling/sampling_logp_difference/mean": 0.024461492896080017,
|
|
"step": 24,
|
|
"step_time": 296.57605882082134
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.041015625,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2020.0,
|
|
"completions/mean_length": 700.2181396484375,
|
|
"completions/mean_terminated_length": 598.7772827148438,
|
|
"completions/min_length": 166.0,
|
|
"completions/min_terminated_length": 166.0,
|
|
"entropy": 0.4677532056812197,
|
|
"epoch": 0.36764705882352944,
|
|
"frac_reward_zero_std": 0.3984375,
|
|
"grad_norm": 0.062490854299157274,
|
|
"learning_rate": 2.862266002987244e-06,
|
|
"loss": -0.0119,
|
|
"num_tokens": 29582571.0,
|
|
"reward": 0.611328125,
|
|
"reward_std": 0.4876072406768799,
|
|
"rewards/reward_correctness/mean": 0.611328125,
|
|
"rewards/reward_correctness/std": 0.48760727047920227,
|
|
"sampling/importance_sampling_ratio/max": 2.9679155349731445,
|
|
"sampling/importance_sampling_ratio/mean": 0.5363779067993164,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9733572006225586,
|
|
"sampling/sampling_logp_difference/mean": 0.023312201723456383,
|
|
"step": 25,
|
|
"step_time": 285.94121135165915
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05859375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2558.0,
|
|
"completions/mean_length": 772.5709838867188,
|
|
"completions/mean_terminated_length": 629.4530029296875,
|
|
"completions/min_length": 123.0,
|
|
"completions/min_terminated_length": 123.0,
|
|
"entropy": 0.5492652888642624,
|
|
"epoch": 0.38235294117647056,
|
|
"frac_reward_zero_std": 0.5390625,
|
|
"grad_norm": 0.043192270355031,
|
|
"learning_rate": 2.847672186137282e-06,
|
|
"loss": -0.0108,
|
|
"num_tokens": 30924112.0,
|
|
"reward": 0.5305989980697632,
|
|
"reward_std": 0.49922531843185425,
|
|
"rewards/reward_correctness/mean": 0.5305989384651184,
|
|
"rewards/reward_correctness/std": 0.499225378036499,
|
|
"sampling/importance_sampling_ratio/max": 2.97066068649292,
|
|
"sampling/importance_sampling_ratio/mean": 0.5015259385108948,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9405348300933838,
|
|
"sampling/sampling_logp_difference/mean": 0.0235705878585577,
|
|
"step": 26,
|
|
"step_time": 293.17900502029806
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0611979179084301,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1762.0,
|
|
"completions/mean_length": 741.4765625,
|
|
"completions/mean_terminated_length": 589.5562133789062,
|
|
"completions/min_length": 133.0,
|
|
"completions/min_terminated_length": 133.0,
|
|
"entropy": 0.5579174467129633,
|
|
"epoch": 0.39705882352941174,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.0600763917962412,
|
|
"learning_rate": 2.8323895991589866e-06,
|
|
"loss": -0.024,
|
|
"num_tokens": 32203420.0,
|
|
"reward": 0.5944010615348816,
|
|
"reward_std": 0.49116748571395874,
|
|
"rewards/reward_correctness/mean": 0.5944010615348816,
|
|
"rewards/reward_correctness/std": 0.49116748571395874,
|
|
"sampling/importance_sampling_ratio/max": 2.97564697265625,
|
|
"sampling/importance_sampling_ratio/mean": 0.5418947339057922,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.997067928314209,
|
|
"sampling/sampling_logp_difference/mean": 0.025170018896460533,
|
|
"step": 27,
|
|
"step_time": 290.5366625688039
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0481770858168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1965.0,
|
|
"completions/mean_length": 702.5709838867188,
|
|
"completions/mean_terminated_length": 584.0923461914062,
|
|
"completions/min_length": 149.0,
|
|
"completions/min_terminated_length": 149.0,
|
|
"entropy": 0.5106982593424618,
|
|
"epoch": 0.4117647058823529,
|
|
"frac_reward_zero_std": 0.515625,
|
|
"grad_norm": 0.041126690061418614,
|
|
"learning_rate": 2.8164270309259034e-06,
|
|
"loss": -0.0144,
|
|
"num_tokens": 33424049.0,
|
|
"reward": 0.5846354365348816,
|
|
"reward_std": 0.49294525384902954,
|
|
"rewards/reward_correctness/mean": 0.5846354365348816,
|
|
"rewards/reward_correctness/std": 0.49294528365135193,
|
|
"sampling/importance_sampling_ratio/max": 2.9962897300720215,
|
|
"sampling/importance_sampling_ratio/mean": 0.4718739092350006,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 10.12062931060791,
|
|
"sampling/sampling_logp_difference/mean": 0.041104961186647415,
|
|
"step": 28,
|
|
"step_time": 327.90168132819235
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0716145858168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1842.0,
|
|
"completions/mean_length": 766.5319213867188,
|
|
"completions/mean_terminated_length": 588.6907348632812,
|
|
"completions/min_length": 146.0,
|
|
"completions/min_terminated_length": 146.0,
|
|
"entropy": 0.5984537588665262,
|
|
"epoch": 0.4264705882352941,
|
|
"frac_reward_zero_std": 0.453125,
|
|
"grad_norm": 0.05843225178744546,
|
|
"learning_rate": 2.79979366136247e-06,
|
|
"loss": -0.0207,
|
|
"num_tokens": 34740930.0,
|
|
"reward": 0.560546875,
|
|
"reward_std": 0.49648216366767883,
|
|
"rewards/reward_correctness/mean": 0.560546875,
|
|
"rewards/reward_correctness/std": 0.4964821934700012,
|
|
"sampling/importance_sampling_ratio/max": 2.9964332580566406,
|
|
"sampling/importance_sampling_ratio/mean": 0.5357107520103455,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.1054515838623047,
|
|
"sampling/sampling_logp_difference/mean": 0.026250643655657768,
|
|
"step": 29,
|
|
"step_time": 284.9848792338744
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0494791679084301,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1768.0,
|
|
"completions/mean_length": 686.326171875,
|
|
"completions/mean_terminated_length": 562.62744140625,
|
|
"completions/min_length": 120.0,
|
|
"completions/min_terminated_length": 120.0,
|
|
"entropy": 0.49350566777866334,
|
|
"epoch": 0.4411764705882353,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.04678745046376859,
|
|
"learning_rate": 2.7824990561647276e-06,
|
|
"loss": -0.0076,
|
|
"num_tokens": 35927967.0,
|
|
"reward": 0.6080729365348816,
|
|
"reward_std": 0.48833951354026794,
|
|
"rewards/reward_correctness/mean": 0.6080729365348816,
|
|
"rewards/reward_correctness/std": 0.48833951354026794,
|
|
"sampling/importance_sampling_ratio/max": 2.9987664222717285,
|
|
"sampling/importance_sampling_ratio/mean": 0.5746848583221436,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.1287574768066406,
|
|
"sampling/sampling_logp_difference/mean": 0.02340259775519371,
|
|
"step": 30,
|
|
"step_time": 295.85874052438885
|
|
},
|
|
{
|
|
"epoch": 0.4411764705882353,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.001984127043258576,
|
|
"eval_completions/max_length": 1007.0119047619048,
|
|
"eval_completions/max_terminated_length": 979.1071428571429,
|
|
"eval_completions/mean_length": 562.2936713809058,
|
|
"eval_completions/mean_terminated_length": 557.4218459356399,
|
|
"eval_completions/min_length": 265.85714285714283,
|
|
"eval_completions/min_terminated_length": 265.85714285714283,
|
|
"eval_entropy": 0.22903412624838806,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 35927967.0,
|
|
"eval_reward": 0.7162698643902937,
|
|
"eval_reward_std": 0.4320314891991161,
|
|
"eval_rewards/reward_correctness/mean": 0.7162698402646042,
|
|
"eval_rewards/reward_correctness/std": 0.43203149097306387,
|
|
"eval_runtime": 614.0577,
|
|
"eval_samples_per_second": 0.814,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.22348843947717492,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.04295138852800543,
|
|
"eval_sampling/importance_sampling_ratio/min": 2.4404202079057176e-06,
|
|
"eval_sampling/sampling_logp_difference/max": 2.629038010324751,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.05504328708740927,
|
|
"eval_steps_per_second": 0.137,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.064453125,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1561.0,
|
|
"completions/mean_length": 737.421875,
|
|
"completions/mean_terminated_length": 576.7390747070312,
|
|
"completions/min_length": 147.0,
|
|
"completions/min_terminated_length": 147.0,
|
|
"entropy": 0.48714099975768477,
|
|
"epoch": 0.45588235294117646,
|
|
"frac_reward_zero_std": 0.53125,
|
|
"grad_norm": 0.048461134100390504,
|
|
"learning_rate": 2.7645531612991763e-06,
|
|
"loss": -0.0266,
|
|
"num_tokens": 37204179.0,
|
|
"reward": 0.58203125,
|
|
"reward_std": 0.49338558316230774,
|
|
"rewards/reward_correctness/mean": 0.58203125,
|
|
"rewards/reward_correctness/std": 0.4933856129646301,
|
|
"sampling/importance_sampling_ratio/max": 2.982936382293701,
|
|
"sampling/importance_sampling_ratio/mean": 0.5539844036102295,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.9513862133026123,
|
|
"sampling/sampling_logp_difference/mean": 0.023642152547836304,
|
|
"step": 31,
|
|
"step_time": 292.74457041174173
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0690104216337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1547.0,
|
|
"completions/mean_length": 799.5123901367188,
|
|
"completions/mean_terminated_length": 631.062255859375,
|
|
"completions/min_length": 128.0,
|
|
"completions/min_terminated_length": 128.0,
|
|
"entropy": 0.5004677245160565,
|
|
"epoch": 0.47058823529411764,
|
|
"frac_reward_zero_std": 0.515625,
|
|
"grad_norm": 0.048193616973374806,
|
|
"learning_rate": 2.745966297282944e-06,
|
|
"loss": -0.0023,
|
|
"num_tokens": 38554066.0,
|
|
"reward": 0.564453125,
|
|
"reward_std": 0.4959898591041565,
|
|
"rewards/reward_correctness/mean": 0.564453125,
|
|
"rewards/reward_correctness/std": 0.4959898591041565,
|
|
"sampling/importance_sampling_ratio/max": 2.99735164642334,
|
|
"sampling/importance_sampling_ratio/mean": 0.5355834364891052,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 6.243207931518555,
|
|
"sampling/sampling_logp_difference/mean": 0.02345161698758602,
|
|
"step": 32,
|
|
"step_time": 280.2582347355783
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.076171875,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2621.0,
|
|
"completions/mean_length": 778.2904052734375,
|
|
"completions/mean_terminated_length": 589.1683959960938,
|
|
"completions/min_length": 150.0,
|
|
"completions/min_terminated_length": 150.0,
|
|
"entropy": 0.5261909196851775,
|
|
"epoch": 0.4852941176470588,
|
|
"frac_reward_zero_std": 0.4921875,
|
|
"grad_norm": 0.05577718738565114,
|
|
"learning_rate": 2.726749153248549e-06,
|
|
"loss": -0.0388,
|
|
"num_tokens": 39900384.0,
|
|
"reward": 0.5826823115348816,
|
|
"reward_std": 0.49327683448791504,
|
|
"rewards/reward_correctness/mean": 0.5826823115348816,
|
|
"rewards/reward_correctness/std": 0.49327683448791504,
|
|
"sampling/importance_sampling_ratio/max": 2.9583816528320312,
|
|
"sampling/importance_sampling_ratio/mean": 0.5347757935523987,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9872760772705078,
|
|
"sampling/sampling_logp_difference/mean": 0.026426147669553757,
|
|
"step": 33,
|
|
"step_time": 302.06614747131243
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0611979179084301,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1492.0,
|
|
"completions/mean_length": 722.3021240234375,
|
|
"completions/mean_terminated_length": 569.1824340820312,
|
|
"completions/min_length": 186.0,
|
|
"completions/min_terminated_length": 186.0,
|
|
"entropy": 0.507065630867146,
|
|
"epoch": 0.5,
|
|
"frac_reward_zero_std": 0.5078125,
|
|
"grad_norm": 0.06494124688870825,
|
|
"learning_rate": 2.706912780796687e-06,
|
|
"loss": -0.0477,
|
|
"num_tokens": 41176124.0,
|
|
"reward": 0.6178385615348816,
|
|
"reward_std": 0.48607394099235535,
|
|
"rewards/reward_correctness/mean": 0.6178385615348816,
|
|
"rewards/reward_correctness/std": 0.48607397079467773,
|
|
"sampling/importance_sampling_ratio/max": 2.944589614868164,
|
|
"sampling/importance_sampling_ratio/mean": 0.5623457431793213,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 14.96677017211914,
|
|
"sampling/sampling_logp_difference/mean": 0.024992134422063828,
|
|
"step": 34,
|
|
"step_time": 305.60695807402954
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0677083358168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 3068.0,
|
|
"completions/mean_length": 775.7916870117188,
|
|
"completions/mean_terminated_length": 609.6054077148438,
|
|
"completions/min_length": 148.0,
|
|
"completions/min_terminated_length": 148.0,
|
|
"entropy": 0.4794918481493369,
|
|
"epoch": 0.5147058823529411,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.053901537826546005,
|
|
"learning_rate": 2.686468587640551e-06,
|
|
"loss": -0.0135,
|
|
"num_tokens": 42518328.0,
|
|
"reward": 0.60546875,
|
|
"reward_std": 0.48890891671180725,
|
|
"rewards/reward_correctness/mean": 0.60546875,
|
|
"rewards/reward_correctness/std": 0.48890894651412964,
|
|
"sampling/importance_sampling_ratio/max": 2.9897232055664062,
|
|
"sampling/importance_sampling_ratio/mean": 0.5776769518852234,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.987520694732666,
|
|
"sampling/sampling_logp_difference/mean": 0.023517360910773277,
|
|
"step": 35,
|
|
"step_time": 297.2315682750195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.091796875,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1907.0,
|
|
"completions/mean_length": 819.9349365234375,
|
|
"completions/mean_terminated_length": 593.167724609375,
|
|
"completions/min_length": 151.0,
|
|
"completions/min_terminated_length": 151.0,
|
|
"entropy": 0.5791506089735776,
|
|
"epoch": 0.5294117647058824,
|
|
"frac_reward_zero_std": 0.5234375,
|
|
"grad_norm": 0.05513570710996907,
|
|
"learning_rate": 2.6654283310453644e-06,
|
|
"loss": -0.0357,
|
|
"num_tokens": 43934372.0,
|
|
"reward": 0.5709635615348816,
|
|
"reward_std": 0.4950997233390808,
|
|
"rewards/reward_correctness/mean": 0.5709635615348816,
|
|
"rewards/reward_correctness/std": 0.4950997531414032,
|
|
"sampling/importance_sampling_ratio/max": 2.9943625926971436,
|
|
"sampling/importance_sampling_ratio/mean": 0.5423656105995178,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 2.370652437210083,
|
|
"sampling/sampling_logp_difference/mean": 0.029513418674468994,
|
|
"step": 36,
|
|
"step_time": 310.507658964023
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0813802108168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2379.0,
|
|
"completions/mean_length": 779.7272338867188,
|
|
"completions/mean_terminated_length": 576.839111328125,
|
|
"completions/min_length": 157.0,
|
|
"completions/min_terminated_length": 157.0,
|
|
"entropy": 0.49720528721809387,
|
|
"epoch": 0.5441176470588235,
|
|
"frac_reward_zero_std": 0.5234375,
|
|
"grad_norm": 0.046276621163544786,
|
|
"learning_rate": 2.643804111066888e-06,
|
|
"loss": -0.0157,
|
|
"num_tokens": 45276405.0,
|
|
"reward": 0.6263021230697632,
|
|
"reward_std": 0.4839423894882202,
|
|
"rewards/reward_correctness/mean": 0.6263020634651184,
|
|
"rewards/reward_correctness/std": 0.4839424192905426,
|
|
"sampling/importance_sampling_ratio/max": 2.992263078689575,
|
|
"sampling/importance_sampling_ratio/mean": 0.5235354900360107,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.8799822330474854,
|
|
"sampling/sampling_logp_difference/mean": 0.026051774621009827,
|
|
"step": 37,
|
|
"step_time": 311.45988411689177
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0872395858168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1678.0,
|
|
"completions/mean_length": 790.8515625,
|
|
"completions/mean_terminated_length": 572.8245239257812,
|
|
"completions/min_length": 112.0,
|
|
"completions/min_terminated_length": 112.0,
|
|
"entropy": 0.5321638238383457,
|
|
"epoch": 0.5588235294117647,
|
|
"frac_reward_zero_std": 0.53125,
|
|
"grad_norm": 0.0379364986328415,
|
|
"learning_rate": 2.6216083635927896e-06,
|
|
"loss": -0.0109,
|
|
"num_tokens": 46624845.0,
|
|
"reward": 0.5638021230697632,
|
|
"reward_std": 0.4960740804672241,
|
|
"rewards/reward_correctness/mean": 0.5638020634651184,
|
|
"rewards/reward_correctness/std": 0.4960740804672241,
|
|
"sampling/importance_sampling_ratio/max": 2.985919713973999,
|
|
"sampling/importance_sampling_ratio/mean": 0.5088026523590088,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.3098070621490479,
|
|
"sampling/sampling_logp_difference/mean": 0.028229426592588425,
|
|
"step": 38,
|
|
"step_time": 298.7274723085575
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.083984375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2886.0,
|
|
"completions/mean_length": 789.1823120117188,
|
|
"completions/mean_terminated_length": 579.9559326171875,
|
|
"completions/min_length": 120.0,
|
|
"completions/min_terminated_length": 120.0,
|
|
"entropy": 0.48998611723072827,
|
|
"epoch": 0.5735294117647058,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.05913528982233647,
|
|
"learning_rate": 2.598853853190882e-06,
|
|
"loss": -0.0118,
|
|
"num_tokens": 47975881.0,
|
|
"reward": 0.6067708730697632,
|
|
"reward_std": 0.48862606287002563,
|
|
"rewards/reward_correctness/mean": 0.6067708134651184,
|
|
"rewards/reward_correctness/std": 0.48862606287002563,
|
|
"sampling/importance_sampling_ratio/max": 2.999361991882324,
|
|
"sampling/importance_sampling_ratio/mean": 0.540636420249939,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.0663793087005615,
|
|
"sampling/sampling_logp_difference/mean": 0.026590874418616295,
|
|
"step": 39,
|
|
"step_time": 303.23120561568066
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1028645858168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2553.0,
|
|
"completions/mean_length": 825.0299682617188,
|
|
"completions/mean_terminated_length": 567.4934692382812,
|
|
"completions/min_length": 158.0,
|
|
"completions/min_terminated_length": 158.0,
|
|
"entropy": 0.5337136797606945,
|
|
"epoch": 0.5882352941176471,
|
|
"frac_reward_zero_std": 0.5390625,
|
|
"grad_norm": 0.0500404243931231,
|
|
"learning_rate": 2.5755536657683354e-06,
|
|
"loss": -0.0209,
|
|
"num_tokens": 49384103.0,
|
|
"reward": 0.6022135615348816,
|
|
"reward_std": 0.489600270986557,
|
|
"rewards/reward_correctness/mean": 0.6022135615348816,
|
|
"rewards/reward_correctness/std": 0.4896003007888794,
|
|
"sampling/importance_sampling_ratio/max": 2.989989757537842,
|
|
"sampling/importance_sampling_ratio/mean": 0.5711022019386292,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9798126220703125,
|
|
"sampling/sampling_logp_difference/mean": 0.02729925513267517,
|
|
"step": 40,
|
|
"step_time": 302.83849718095735
|
|
},
|
|
{
|
|
"epoch": 0.5882352941176471,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.011904762259551458,
|
|
"eval_completions/max_length": 1071.4404761904761,
|
|
"eval_completions/max_terminated_length": 927.4404761904761,
|
|
"eval_completions/mean_length": 575.2956495739165,
|
|
"eval_completions/mean_terminated_length": 546.4504107520694,
|
|
"eval_completions/min_length": 274.1904761904762,
|
|
"eval_completions/min_terminated_length": 274.1904761904762,
|
|
"eval_entropy": 0.20750312196711698,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 49384103.0,
|
|
"eval_reward": 0.7103174795352277,
|
|
"eval_reward_std": 0.40739349524180096,
|
|
"eval_rewards/reward_correctness/mean": 0.7103174589574337,
|
|
"eval_rewards/reward_correctness/std": 0.40739349524180096,
|
|
"eval_runtime": 658.6168,
|
|
"eval_samples_per_second": 0.759,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.19148892058893535,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.03784746756954995,
|
|
"eval_sampling/importance_sampling_ratio/min": 6.423417091128806e-05,
|
|
"eval_sampling/sampling_logp_difference/max": 2.551057347229549,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.05068082945598733,
|
|
"eval_steps_per_second": 0.128,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1022135466337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2139.0,
|
|
"completions/mean_length": 836.333984375,
|
|
"completions/mean_terminated_length": 581.802001953125,
|
|
"completions/min_length": 164.0,
|
|
"completions/min_terminated_length": 164.0,
|
|
"entropy": 0.515403852565214,
|
|
"epoch": 0.6029411764705882,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.05273673949955187,
|
|
"learning_rate": 2.551721201046098e-06,
|
|
"loss": -0.0233,
|
|
"num_tokens": 50811980.0,
|
|
"reward": 0.5833333730697632,
|
|
"reward_std": 0.4931671619415283,
|
|
"rewards/reward_correctness/mean": 0.5833333134651184,
|
|
"rewards/reward_correctness/std": 0.4931671917438507,
|
|
"sampling/importance_sampling_ratio/max": 2.9984641075134277,
|
|
"sampling/importance_sampling_ratio/mean": 0.528105616569519,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.419299840927124,
|
|
"sampling/sampling_logp_difference/mean": 0.02731555700302124,
|
|
"step": 41,
|
|
"step_time": 298.77585212513804
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1971.0,
|
|
"completions/mean_length": 858.44140625,
|
|
"completions/mean_terminated_length": 608.2130737304688,
|
|
"completions/min_length": 135.0,
|
|
"completions/min_terminated_length": 135.0,
|
|
"entropy": 0.47979187313467264,
|
|
"epoch": 0.6176470588235294,
|
|
"frac_reward_zero_std": 0.5078125,
|
|
"grad_norm": 0.04893596319312678,
|
|
"learning_rate": 2.5273701648528393e-06,
|
|
"loss": -0.0332,
|
|
"num_tokens": 52270286.0,
|
|
"reward": 0.615234375,
|
|
"reward_std": 0.48669832944869995,
|
|
"rewards/reward_correctness/mean": 0.615234375,
|
|
"rewards/reward_correctness/std": 0.48669832944869995,
|
|
"sampling/importance_sampling_ratio/max": 2.9960122108459473,
|
|
"sampling/importance_sampling_ratio/mean": 0.5512368083000183,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.9809277057647705,
|
|
"sampling/sampling_logp_difference/mean": 0.02453477308154106,
|
|
"step": 42,
|
|
"step_time": 301.4260444557294
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1002604216337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2852.0,
|
|
"completions/mean_length": 823.6107177734375,
|
|
"completions/mean_terminated_length": 573.2756958007812,
|
|
"completions/min_length": 92.0,
|
|
"completions/min_terminated_length": 92.0,
|
|
"entropy": 0.5395284099504352,
|
|
"epoch": 0.6323529411764706,
|
|
"frac_reward_zero_std": 0.4765625,
|
|
"grad_norm": 0.06291468578251484,
|
|
"learning_rate": 2.5025145612428566e-06,
|
|
"loss": -0.0288,
|
|
"num_tokens": 53690056.0,
|
|
"reward": 0.6158854365348816,
|
|
"reward_std": 0.48654359579086304,
|
|
"rewards/reward_correctness/mean": 0.6158854365348816,
|
|
"rewards/reward_correctness/std": 0.48654359579086304,
|
|
"sampling/importance_sampling_ratio/max": 2.9952244758605957,
|
|
"sampling/importance_sampling_ratio/mean": 0.5603159070014954,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9814343452453613,
|
|
"sampling/sampling_logp_difference/mean": 0.02832927368581295,
|
|
"step": 43,
|
|
"step_time": 310.514519975055
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12109375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1776.0,
|
|
"completions/mean_length": 891.1302490234375,
|
|
"completions/mean_terminated_length": 591.8214721679688,
|
|
"completions/min_length": 145.0,
|
|
"completions/min_terminated_length": 145.0,
|
|
"entropy": 0.5378130340250209,
|
|
"epoch": 0.6470588235294118,
|
|
"frac_reward_zero_std": 0.4609375,
|
|
"grad_norm": 0.043823548874597246,
|
|
"learning_rate": 2.47716868444247e-06,
|
|
"loss": -0.0273,
|
|
"num_tokens": 55219440.0,
|
|
"reward": 0.5872396230697632,
|
|
"reward_std": 0.4924907386302948,
|
|
"rewards/reward_correctness/mean": 0.5872395634651184,
|
|
"rewards/reward_correctness/std": 0.4924907684326172,
|
|
"sampling/importance_sampling_ratio/max": 2.9666354656219482,
|
|
"sampling/importance_sampling_ratio/mean": 0.5122290253639221,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9179315567016602,
|
|
"sampling/sampling_logp_difference/mean": 0.02752448432147503,
|
|
"step": 44,
|
|
"step_time": 320.2574398214929
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1041666716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2330.0,
|
|
"completions/mean_length": 843.5462646484375,
|
|
"completions/mean_terminated_length": 584.4927368164062,
|
|
"completions/min_length": 130.0,
|
|
"completions/min_terminated_length": 130.0,
|
|
"entropy": 0.48280362505465746,
|
|
"epoch": 0.6617647058823529,
|
|
"frac_reward_zero_std": 0.5078125,
|
|
"grad_norm": 0.04884731721807342,
|
|
"learning_rate": 2.4513471106295523e-06,
|
|
"loss": -0.0239,
|
|
"num_tokens": 56670071.0,
|
|
"reward": 0.6197916865348816,
|
|
"reward_std": 0.4855959713459015,
|
|
"rewards/reward_correctness/mean": 0.6197916865348816,
|
|
"rewards/reward_correctness/std": 0.4855960011482239,
|
|
"sampling/importance_sampling_ratio/max": 2.9755513668060303,
|
|
"sampling/importance_sampling_ratio/mean": 0.552994430065155,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.006209373474121,
|
|
"sampling/sampling_logp_difference/mean": 0.02560652606189251,
|
|
"step": 45,
|
|
"step_time": 310.76242945715785
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1380208432674408,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2546.0,
|
|
"completions/mean_length": 938.2428588867188,
|
|
"completions/mean_terminated_length": 596.5838623046875,
|
|
"completions/min_length": 151.0,
|
|
"completions/min_terminated_length": 151.0,
|
|
"entropy": 0.575145949027501,
|
|
"epoch": 0.6764705882352942,
|
|
"frac_reward_zero_std": 0.5078125,
|
|
"grad_norm": 0.045178072823470854,
|
|
"learning_rate": 2.4250646895508992e-06,
|
|
"loss": -0.0243,
|
|
"num_tokens": 58269528.0,
|
|
"reward": 0.578125,
|
|
"reward_std": 0.49401959776878357,
|
|
"rewards/reward_correctness/mean": 0.578125,
|
|
"rewards/reward_correctness/std": 0.49401959776878357,
|
|
"sampling/importance_sampling_ratio/max": 2.972738027572632,
|
|
"sampling/importance_sampling_ratio/mean": 0.5216982960700989,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.1360305547714233,
|
|
"sampling/sampling_logp_difference/mean": 0.02904244139790535,
|
|
"step": 46,
|
|
"step_time": 319.69419858045876
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.103515625,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2219.0,
|
|
"completions/mean_length": 843.1536865234375,
|
|
"completions/mean_terminated_length": 588.5592041015625,
|
|
"completions/min_length": 131.0,
|
|
"completions/min_terminated_length": 131.0,
|
|
"entropy": 0.47189394827000797,
|
|
"epoch": 0.6911764705882353,
|
|
"frac_reward_zero_std": 0.4140625,
|
|
"grad_norm": 0.05207771800211288,
|
|
"learning_rate": 2.3983365359822804e-06,
|
|
"loss": -0.0358,
|
|
"num_tokens": 59714840.0,
|
|
"reward": 0.6100260615348816,
|
|
"reward_std": 0.4879028797149658,
|
|
"rewards/reward_correctness/mean": 0.6100260615348816,
|
|
"rewards/reward_correctness/std": 0.4879029095172882,
|
|
"sampling/importance_sampling_ratio/max": 2.9499571323394775,
|
|
"sampling/importance_sampling_ratio/mean": 0.48770689964294434,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 19.950422286987305,
|
|
"sampling/sampling_logp_difference/mean": 0.04360029101371765,
|
|
"step": 47,
|
|
"step_time": 336.6966767683625
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1432291716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2586.0,
|
|
"completions/mean_length": 949.4993896484375,
|
|
"completions/mean_terminated_length": 594.6740112304688,
|
|
"completions/min_length": 144.0,
|
|
"completions/min_terminated_length": 144.0,
|
|
"entropy": 0.5676321291830391,
|
|
"epoch": 0.7058823529411765,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.05296078839838602,
|
|
"learning_rate": 2.3711780210360726e-06,
|
|
"loss": -0.0401,
|
|
"num_tokens": 61317331.0,
|
|
"reward": 0.5950521230697632,
|
|
"reward_std": 0.49104180932044983,
|
|
"rewards/reward_correctness/mean": 0.5950520634651184,
|
|
"rewards/reward_correctness/std": 0.4910418391227722,
|
|
"sampling/importance_sampling_ratio/max": 2.9128077030181885,
|
|
"sampling/importance_sampling_ratio/mean": 0.5409334897994995,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 2.5000057220458984,
|
|
"sampling/sampling_logp_difference/mean": 0.027828708291053772,
|
|
"step": 48,
|
|
"step_time": 331.9593873512931
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1315104216337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1644.0,
|
|
"completions/mean_length": 907.8952026367188,
|
|
"completions/mean_terminated_length": 580.3995361328125,
|
|
"completions/min_length": 170.0,
|
|
"completions/min_terminated_length": 170.0,
|
|
"entropy": 0.5398647881811485,
|
|
"epoch": 0.7205882352941176,
|
|
"frac_reward_zero_std": 0.5234375,
|
|
"grad_norm": 0.052468022399943345,
|
|
"learning_rate": 2.343604763321476e-06,
|
|
"loss": -0.0243,
|
|
"num_tokens": 62849270.0,
|
|
"reward": 0.5716146230697632,
|
|
"reward_std": 0.49500593543052673,
|
|
"rewards/reward_correctness/mean": 0.5716145634651184,
|
|
"rewards/reward_correctness/std": 0.4950059652328491,
|
|
"sampling/importance_sampling_ratio/max": 2.9808614253997803,
|
|
"sampling/importance_sampling_ratio/mean": 0.5400270223617554,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.333378791809082,
|
|
"sampling/sampling_logp_difference/mean": 0.027292978018522263,
|
|
"step": 49,
|
|
"step_time": 322.7655723937787
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.134765625,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2113.0,
|
|
"completions/mean_length": 896.2552490234375,
|
|
"completions/mean_terminated_length": 557.3694458007812,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 0.5093316873535514,
|
|
"epoch": 0.7352941176470589,
|
|
"frac_reward_zero_std": 0.515625,
|
|
"grad_norm": 0.03900787430902907,
|
|
"learning_rate": 2.3156326199623965e-06,
|
|
"loss": -0.0262,
|
|
"num_tokens": 64360474.0,
|
|
"reward": 0.6158854365348816,
|
|
"reward_std": 0.4865436255931854,
|
|
"rewards/reward_correctness/mean": 0.6158854365348816,
|
|
"rewards/reward_correctness/std": 0.48654359579086304,
|
|
"sampling/importance_sampling_ratio/max": 2.997588872909546,
|
|
"sampling/importance_sampling_ratio/mean": 0.5666263699531555,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9845733642578125,
|
|
"sampling/sampling_logp_difference/mean": 0.028033524751663208,
|
|
"step": 50,
|
|
"step_time": 332.36779574258253
|
|
},
|
|
{
|
|
"epoch": 0.7352941176470589,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.00992063521629288,
|
|
"eval_completions/max_length": 1090.095238095238,
|
|
"eval_completions/max_terminated_length": 980.0952380952381,
|
|
"eval_completions/mean_length": 577.4623176938012,
|
|
"eval_completions/mean_terminated_length": 552.9960454305013,
|
|
"eval_completions/min_length": 264.57142857142856,
|
|
"eval_completions/min_terminated_length": 264.57142857142856,
|
|
"eval_entropy": 0.20710848183149383,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 64360474.0,
|
|
"eval_reward": 0.6746031936435473,
|
|
"eval_reward_std": 0.45623941009952906,
|
|
"eval_rewards/reward_correctness/mean": 0.6746031773232278,
|
|
"eval_rewards/reward_correctness/std": 0.45623940797079177,
|
|
"eval_runtime": 669.8868,
|
|
"eval_samples_per_second": 0.746,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.2080012608648325,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.041787284431111835,
|
|
"eval_sampling/importance_sampling_ratio/min": 1.1791101893997088e-05,
|
|
"eval_sampling/sampling_logp_difference/max": 2.6629502290771123,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.05015929979050443,
|
|
"eval_steps_per_second": 0.125,
|
|
"step": 50
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1354166716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2179.0,
|
|
"completions/mean_length": 901.7982177734375,
|
|
"completions/mean_terminated_length": 561.8870239257812,
|
|
"completions/min_length": 141.0,
|
|
"completions/min_terminated_length": 141.0,
|
|
"entropy": 0.5385016986401752,
|
|
"epoch": 0.75,
|
|
"frac_reward_zero_std": 0.4296875,
|
|
"grad_norm": 0.0426987144482271,
|
|
"learning_rate": 2.2872776774781627e-06,
|
|
"loss": -0.024,
|
|
"num_tokens": 65872596.0,
|
|
"reward": 0.6282552480697632,
|
|
"reward_std": 0.48342809081077576,
|
|
"rewards/reward_correctness/mean": 0.6282551884651184,
|
|
"rewards/reward_correctness/std": 0.48342812061309814,
|
|
"sampling/importance_sampling_ratio/max": 2.981839895248413,
|
|
"sampling/importance_sampling_ratio/mean": 0.5335763096809387,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.6584514379501343,
|
|
"sampling/sampling_logp_difference/mean": 0.028055667877197266,
|
|
"step": 51,
|
|
"step_time": 300.4838323034346
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1302083432674408,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2621.0,
|
|
"completions/mean_length": 904.3053588867188,
|
|
"completions/mean_terminated_length": 580.3997192382812,
|
|
"completions/min_length": 144.0,
|
|
"completions/min_terminated_length": 144.0,
|
|
"entropy": 0.5088172309333459,
|
|
"epoch": 0.7647058823529411,
|
|
"frac_reward_zero_std": 0.5078125,
|
|
"grad_norm": 0.04766427212983689,
|
|
"learning_rate": 2.258556242532317e-06,
|
|
"loss": -0.0177,
|
|
"num_tokens": 67414141.0,
|
|
"reward": 0.57421875,
|
|
"reward_std": 0.4946219325065613,
|
|
"rewards/reward_correctness/mean": 0.57421875,
|
|
"rewards/reward_correctness/std": 0.4946219325065613,
|
|
"sampling/importance_sampling_ratio/max": 2.9827163219451904,
|
|
"sampling/importance_sampling_ratio/mean": 0.48218855261802673,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 32.999961853027344,
|
|
"sampling/sampling_logp_difference/mean": 0.04324951022863388,
|
|
"step": 52,
|
|
"step_time": 343.32239248557016
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1686197966337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2094.0,
|
|
"completions/mean_length": 1033.1204833984375,
|
|
"completions/mean_terminated_length": 619.5967407226562,
|
|
"completions/min_length": 122.0,
|
|
"completions/min_terminated_length": 122.0,
|
|
"entropy": 0.5954927550628781,
|
|
"epoch": 0.7794117647058824,
|
|
"frac_reward_zero_std": 0.4296875,
|
|
"grad_norm": 0.03832772147605136,
|
|
"learning_rate": 2.2294848325548066e-06,
|
|
"loss": -0.0349,
|
|
"num_tokens": 69140778.0,
|
|
"reward": 0.5305989980697632,
|
|
"reward_std": 0.49922534823417664,
|
|
"rewards/reward_correctness/mean": 0.5305989384651184,
|
|
"rewards/reward_correctness/std": 0.499225378036499,
|
|
"sampling/importance_sampling_ratio/max": 2.947634696960449,
|
|
"sampling/importance_sampling_ratio/mean": 0.4914858043193817,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.1630563735961914,
|
|
"sampling/sampling_logp_difference/mean": 0.02942274883389473,
|
|
"step": 53,
|
|
"step_time": 320.89451397629455
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1139322966337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2290.0,
|
|
"completions/mean_length": 832.9329833984375,
|
|
"completions/mean_terminated_length": 545.0293579101562,
|
|
"completions/min_length": 142.0,
|
|
"completions/min_terminated_length": 142.0,
|
|
"entropy": 0.45392935909330845,
|
|
"epoch": 0.7941176470588235,
|
|
"frac_reward_zero_std": 0.453125,
|
|
"grad_norm": 0.05567566279138782,
|
|
"learning_rate": 2.200080166242961e-06,
|
|
"loss": -0.0314,
|
|
"num_tokens": 70557443.0,
|
|
"reward": 0.6595052480697632,
|
|
"reward_std": 0.47402989864349365,
|
|
"rewards/reward_correctness/mean": 0.6595051884651184,
|
|
"rewards/reward_correctness/std": 0.4740299582481384,
|
|
"sampling/importance_sampling_ratio/max": 2.9649579524993896,
|
|
"sampling/importance_sampling_ratio/mean": 0.576991856098175,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 4.315248012542725,
|
|
"sampling/sampling_logp_difference/mean": 0.02493918500840664,
|
|
"step": 54,
|
|
"step_time": 313.9590461105108
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.18359375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2932.0,
|
|
"completions/mean_length": 1055.672607421875,
|
|
"completions/mean_terminated_length": 602.5557861328125,
|
|
"completions/min_length": 149.0,
|
|
"completions/min_terminated_length": 149.0,
|
|
"entropy": 0.6210943238111213,
|
|
"epoch": 0.8088235294117647,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.04392399703168934,
|
|
"learning_rate": 2.1703591539467283e-06,
|
|
"loss": -0.0246,
|
|
"num_tokens": 72337356.0,
|
|
"reward": 0.5475260615348816,
|
|
"reward_std": 0.49789825081825256,
|
|
"rewards/reward_correctness/mean": 0.5475260615348816,
|
|
"rewards/reward_correctness/std": 0.49789825081825256,
|
|
"sampling/importance_sampling_ratio/max": 2.961296796798706,
|
|
"sampling/importance_sampling_ratio/mean": 0.512884795665741,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9878766536712646,
|
|
"sampling/sampling_logp_difference/mean": 0.029556401073932648,
|
|
"step": 55,
|
|
"step_time": 341.59292052686214
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.154296875,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2412.0,
|
|
"completions/mean_length": 957.0729370117188,
|
|
"completions/mean_terminated_length": 571.8860473632812,
|
|
"completions/min_length": 130.0,
|
|
"completions/min_terminated_length": 130.0,
|
|
"entropy": 0.5751702012494206,
|
|
"epoch": 0.8235294117647058,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.04530116895001296,
|
|
"learning_rate": 2.140338887943686e-06,
|
|
"loss": -0.0141,
|
|
"num_tokens": 73948192.0,
|
|
"reward": 0.6041666865348816,
|
|
"reward_std": 0.48918819427490234,
|
|
"rewards/reward_correctness/mean": 0.6041666865348816,
|
|
"rewards/reward_correctness/std": 0.48918819427490234,
|
|
"sampling/importance_sampling_ratio/max": 2.985471487045288,
|
|
"sampling/importance_sampling_ratio/mean": 0.5680562853813171,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.0761712789535522,
|
|
"sampling/sampling_logp_difference/mean": 0.029566694051027298,
|
|
"step": 56,
|
|
"step_time": 330.4778544139117
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1302083432674408,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2915.0,
|
|
"completions/mean_length": 892.671875,
|
|
"completions/mean_terminated_length": 566.4251708984375,
|
|
"completions/min_length": 104.0,
|
|
"completions/min_terminated_length": 104.0,
|
|
"entropy": 0.49189388286322355,
|
|
"epoch": 0.8382352941176471,
|
|
"frac_reward_zero_std": 0.4453125,
|
|
"grad_norm": 0.04280765723311196,
|
|
"learning_rate": 2.110036632609435e-06,
|
|
"loss": -0.0408,
|
|
"num_tokens": 75445696.0,
|
|
"reward": 0.6490885615348816,
|
|
"reward_std": 0.4774106740951538,
|
|
"rewards/reward_correctness/mean": 0.6490885615348816,
|
|
"rewards/reward_correctness/std": 0.4774107038974762,
|
|
"sampling/importance_sampling_ratio/max": 2.9966816902160645,
|
|
"sampling/importance_sampling_ratio/mean": 0.5869173407554626,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 2.199824333190918,
|
|
"sampling/sampling_logp_difference/mean": 0.02581387758255005,
|
|
"step": 57,
|
|
"step_time": 312.6213083774783
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1666666716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 3065.0,
|
|
"completions/mean_length": 1012.7077026367188,
|
|
"completions/mean_terminated_length": 606.9664306640625,
|
|
"completions/min_length": 134.0,
|
|
"completions/min_terminated_length": 134.0,
|
|
"entropy": 0.609487400855869,
|
|
"epoch": 0.8529411764705882,
|
|
"frac_reward_zero_std": 0.4921875,
|
|
"grad_norm": 0.04534803213910246,
|
|
"learning_rate": 2.0794698144890156e-06,
|
|
"loss": -0.045,
|
|
"num_tokens": 77166719.0,
|
|
"reward": 0.5846354365348816,
|
|
"reward_std": 0.49294525384902954,
|
|
"rewards/reward_correctness/mean": 0.5846354365348816,
|
|
"rewards/reward_correctness/std": 0.49294528365135193,
|
|
"sampling/importance_sampling_ratio/max": 2.991367816925049,
|
|
"sampling/importance_sampling_ratio/mean": 0.44743868708610535,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 21.9999942779541,
|
|
"sampling/sampling_logp_difference/mean": 0.04774170368909836,
|
|
"step": 58,
|
|
"step_time": 348.07748995441943
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1751302182674408,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2943.0,
|
|
"completions/mean_length": 1030.2064208984375,
|
|
"completions/mean_terminated_length": 597.9454956054688,
|
|
"completions/min_length": 157.0,
|
|
"completions/min_terminated_length": 157.0,
|
|
"entropy": 0.6143204279942438,
|
|
"epoch": 0.8676470588235294,
|
|
"frac_reward_zero_std": 0.4609375,
|
|
"grad_norm": 0.04810753389214618,
|
|
"learning_rate": 2.048656012275064e-06,
|
|
"loss": -0.0201,
|
|
"num_tokens": 78909004.0,
|
|
"reward": 0.5247396230697632,
|
|
"reward_std": 0.4995501935482025,
|
|
"rewards/reward_correctness/mean": 0.5247395634651184,
|
|
"rewards/reward_correctness/std": 0.4995502233505249,
|
|
"sampling/importance_sampling_ratio/max": 2.9893922805786133,
|
|
"sampling/importance_sampling_ratio/mean": 0.5370829105377197,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.0024757385253906,
|
|
"sampling/sampling_logp_difference/mean": 0.030788302421569824,
|
|
"step": 59,
|
|
"step_time": 352.94321115035564
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1979166716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2871.0,
|
|
"completions/mean_length": 1106.5794677734375,
|
|
"completions/mean_terminated_length": 621.6055297851562,
|
|
"completions/min_length": 111.0,
|
|
"completions/min_terminated_length": 111.0,
|
|
"entropy": 0.6316488899756223,
|
|
"epoch": 0.8823529411764706,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.058038002186570306,
|
|
"learning_rate": 2.017612946698471e-06,
|
|
"loss": -0.0359,
|
|
"num_tokens": 80753634.0,
|
|
"reward": 0.5325521230697632,
|
|
"reward_std": 0.4991016983985901,
|
|
"rewards/reward_correctness/mean": 0.5325520634651184,
|
|
"rewards/reward_correctness/std": 0.4991017282009125,
|
|
"sampling/importance_sampling_ratio/max": 2.9939475059509277,
|
|
"sampling/importance_sampling_ratio/mean": 0.525791347026825,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.1060563325881958,
|
|
"sampling/sampling_logp_difference/mean": 0.030230939388275146,
|
|
"step": 60,
|
|
"step_time": 357.5172311188653
|
|
},
|
|
{
|
|
"epoch": 0.8823529411764706,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.033730159735395795,
|
|
"eval_completions/max_length": 1374.0119047619048,
|
|
"eval_completions/max_terminated_length": 942.7857142857143,
|
|
"eval_completions/mean_length": 636.5297782534644,
|
|
"eval_completions/mean_terminated_length": 549.8853305634998,
|
|
"eval_completions/min_length": 268.7857142857143,
|
|
"eval_completions/min_terminated_length": 268.7857142857143,
|
|
"eval_entropy": 0.20073904212386834,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 80753634.0,
|
|
"eval_reward": 0.7003968453833035,
|
|
"eval_reward_std": 0.4480584969832784,
|
|
"eval_rewards/reward_correctness/mean": 0.700396828353405,
|
|
"eval_rewards/reward_correctness/std": 0.4480584930805933,
|
|
"eval_runtime": 858.7508,
|
|
"eval_samples_per_second": 0.582,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.20012787568356588,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.042331399595457786,
|
|
"eval_sampling/importance_sampling_ratio/min": 6.596829363872345e-05,
|
|
"eval_sampling/sampling_logp_difference/max": 2.5949947152818953,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.048185367846772784,
|
|
"eval_steps_per_second": 0.098,
|
|
"step": 60
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1731770932674408,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2855.0,
|
|
"completions/mean_length": 1015.671875,
|
|
"completions/mean_terminated_length": 585.7787475585938,
|
|
"completions/min_length": 150.0,
|
|
"completions/min_terminated_length": 150.0,
|
|
"entropy": 0.6050877087982371,
|
|
"epoch": 0.8970588235294118,
|
|
"frac_reward_zero_std": 0.4375,
|
|
"grad_norm": 0.054576992161028744,
|
|
"learning_rate": 1.9863584703373534e-06,
|
|
"loss": -0.0424,
|
|
"num_tokens": 82474254.0,
|
|
"reward": 0.5514323115348816,
|
|
"reward_std": 0.497509628534317,
|
|
"rewards/reward_correctness/mean": 0.5514323115348816,
|
|
"rewards/reward_correctness/std": 0.497509628534317,
|
|
"sampling/importance_sampling_ratio/max": 2.995573043823242,
|
|
"sampling/importance_sampling_ratio/mean": 0.5062848925590515,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 5.065879821777344,
|
|
"sampling/sampling_logp_difference/mean": 0.030388731509447098,
|
|
"step": 61,
|
|
"step_time": 331.8993032807484
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1725260466337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2556.0,
|
|
"completions/mean_length": 1073.9447021484375,
|
|
"completions/mean_terminated_length": 658.0574340820312,
|
|
"completions/min_length": 131.0,
|
|
"completions/min_terminated_length": 131.0,
|
|
"entropy": 0.5703155723167583,
|
|
"epoch": 0.9117647058823529,
|
|
"frac_reward_zero_std": 0.390625,
|
|
"grad_norm": 0.05019150172655544,
|
|
"learning_rate": 1.954910557350202e-06,
|
|
"loss": -0.0448,
|
|
"num_tokens": 84280817.0,
|
|
"reward": 0.6087239980697632,
|
|
"reward_std": 0.48819488286972046,
|
|
"rewards/reward_correctness/mean": 0.6087239384651184,
|
|
"rewards/reward_correctness/std": 0.48819491267204285,
|
|
"sampling/importance_sampling_ratio/max": 2.9991607666015625,
|
|
"sampling/importance_sampling_ratio/mean": 0.5425732731819153,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.4889273643493652,
|
|
"sampling/sampling_logp_difference/mean": 0.028018062934279442,
|
|
"step": 62,
|
|
"step_time": 341.14307256788015
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1653645932674408,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2430.0,
|
|
"completions/mean_length": 1036.990966796875,
|
|
"completions/mean_terminated_length": 633.828369140625,
|
|
"completions/min_length": 161.0,
|
|
"completions/min_terminated_length": 161.0,
|
|
"entropy": 0.6102959946729243,
|
|
"epoch": 0.9264705882352942,
|
|
"frac_reward_zero_std": 0.3828125,
|
|
"grad_norm": 0.0804020292899743,
|
|
"learning_rate": 1.9232872931391114e-06,
|
|
"loss": -0.028,
|
|
"num_tokens": 86030823.0,
|
|
"reward": 0.4895833432674408,
|
|
"reward_std": 0.5000542402267456,
|
|
"rewards/reward_correctness/mean": 0.4895833432674408,
|
|
"rewards/reward_correctness/std": 0.5000542402267456,
|
|
"sampling/importance_sampling_ratio/max": 2.9987845420837402,
|
|
"sampling/importance_sampling_ratio/mean": 0.527662456035614,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.5172169208526611,
|
|
"sampling/sampling_logp_difference/mean": 0.030840862542390823,
|
|
"step": 63,
|
|
"step_time": 323.75026592426
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.154296875,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2693.0,
|
|
"completions/mean_length": 976.2330932617188,
|
|
"completions/mean_terminated_length": 595.9468383789062,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"entropy": 0.5973206992493942,
|
|
"epoch": 0.9411764705882353,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.0563704880498028,
|
|
"learning_rate": 1.8915068639490344e-06,
|
|
"loss": -0.0213,
|
|
"num_tokens": 87680029.0,
|
|
"reward": 0.5852864980697632,
|
|
"reward_std": 0.4928329586982727,
|
|
"rewards/reward_correctness/mean": 0.5852864384651184,
|
|
"rewards/reward_correctness/std": 0.4928329586982727,
|
|
"sampling/importance_sampling_ratio/max": 2.9954569339752197,
|
|
"sampling/importance_sampling_ratio/mean": 0.4369260370731354,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 18.054588317871094,
|
|
"sampling/sampling_logp_difference/mean": 0.04849446564912796,
|
|
"step": 64,
|
|
"step_time": 339.74727647705004
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15234375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2495.0,
|
|
"completions/mean_length": 966.34375,
|
|
"completions/mean_terminated_length": 588.47314453125,
|
|
"completions/min_length": 155.0,
|
|
"completions/min_terminated_length": 155.0,
|
|
"entropy": 0.6048457206925377,
|
|
"epoch": 0.9558823529411765,
|
|
"frac_reward_zero_std": 0.4140625,
|
|
"grad_norm": 0.04591077537993929,
|
|
"learning_rate": 1.8595875464090389e-06,
|
|
"loss": -0.0417,
|
|
"num_tokens": 89316793.0,
|
|
"reward": 0.6009114980697632,
|
|
"reward_std": 0.48987048864364624,
|
|
"rewards/reward_correctness/mean": 0.6009114384651184,
|
|
"rewards/reward_correctness/std": 0.489870548248291,
|
|
"sampling/importance_sampling_ratio/max": 2.9966180324554443,
|
|
"sampling/importance_sampling_ratio/mean": 0.4967881739139557,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.4514122009277344,
|
|
"sampling/sampling_logp_difference/mean": 0.03166817128658295,
|
|
"step": 65,
|
|
"step_time": 330.3027874575928
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.126953125,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2902.0,
|
|
"completions/mean_length": 915.1068115234375,
|
|
"completions/mean_terminated_length": 601.72705078125,
|
|
"completions/min_length": 125.0,
|
|
"completions/min_terminated_length": 125.0,
|
|
"entropy": 0.5254314645426348,
|
|
"epoch": 0.9705882352941176,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.06999161224416137,
|
|
"learning_rate": 1.8275476970215906e-06,
|
|
"loss": -0.0406,
|
|
"num_tokens": 90877761.0,
|
|
"reward": 0.5625,
|
|
"reward_std": 0.49623993039131165,
|
|
"rewards/reward_correctness/mean": 0.5625,
|
|
"rewards/reward_correctness/std": 0.49623993039131165,
|
|
"sampling/importance_sampling_ratio/max": 2.9245617389678955,
|
|
"sampling/importance_sampling_ratio/mean": 0.5559518337249756,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.640268325805664,
|
|
"sampling/sampling_logp_difference/mean": 0.02776806429028511,
|
|
"step": 66,
|
|
"step_time": 312.4176756278612
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1256510466337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2985.0,
|
|
"completions/mean_length": 934.9114990234375,
|
|
"completions/mean_terminated_length": 627.7944946289062,
|
|
"completions/min_length": 137.0,
|
|
"completions/min_terminated_length": 137.0,
|
|
"entropy": 0.5442105032270774,
|
|
"epoch": 0.9852941176470589,
|
|
"frac_reward_zero_std": 0.484375,
|
|
"grad_norm": 0.05063560268720398,
|
|
"learning_rate": 1.7954057416059002e-06,
|
|
"loss": -0.0278,
|
|
"num_tokens": 92443733.0,
|
|
"reward": 0.5774739980697632,
|
|
"reward_std": 0.49412214756011963,
|
|
"rewards/reward_correctness/mean": 0.5774739384651184,
|
|
"rewards/reward_correctness/std": 0.4941222071647644,
|
|
"sampling/importance_sampling_ratio/max": 2.982328176498413,
|
|
"sampling/importance_sampling_ratio/mean": 0.522100031375885,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.3922834396362305,
|
|
"sampling/sampling_logp_difference/mean": 0.027582859620451927,
|
|
"step": 67,
|
|
"step_time": 306.1620469666086
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1178385466337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2569.0,
|
|
"completions/mean_length": 878.169921875,
|
|
"completions/mean_terminated_length": 585.1195678710938,
|
|
"completions/min_length": 166.0,
|
|
"completions/min_terminated_length": 166.0,
|
|
"entropy": 0.5163884535431862,
|
|
"epoch": 1.0,
|
|
"frac_reward_zero_std": 0.4453125,
|
|
"grad_norm": 0.05421093036584026,
|
|
"learning_rate": 1.7631801647014034e-06,
|
|
"loss": -0.0278,
|
|
"num_tokens": 93931514.0,
|
|
"reward": 0.6119791865348816,
|
|
"reward_std": 0.4874580502510071,
|
|
"rewards/reward_correctness/mean": 0.6119791865348816,
|
|
"rewards/reward_correctness/std": 0.4874580502510071,
|
|
"sampling/importance_sampling_ratio/max": 2.9934051036834717,
|
|
"sampling/importance_sampling_ratio/mean": 0.5653523802757263,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.140676498413086,
|
|
"sampling/sampling_logp_difference/mean": 0.02798476442694664,
|
|
"step": 68,
|
|
"step_time": 295.66030500503257
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1484375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2770.0,
|
|
"completions/mean_length": 971.5078125,
|
|
"completions/mean_terminated_length": 605.4052124023438,
|
|
"completions/min_length": 152.0,
|
|
"completions/min_terminated_length": 152.0,
|
|
"entropy": 0.5773363119224086,
|
|
"epoch": 1.0147058823529411,
|
|
"frac_reward_zero_std": 0.53125,
|
|
"grad_norm": 0.050663113215641147,
|
|
"learning_rate": 1.7308894989374766e-06,
|
|
"loss": -0.0366,
|
|
"num_tokens": 95572970.0,
|
|
"reward": 0.5735677480697632,
|
|
"reward_std": 0.4947192072868347,
|
|
"rewards/reward_correctness/mean": 0.5735676884651184,
|
|
"rewards/reward_correctness/std": 0.4947192370891571,
|
|
"sampling/importance_sampling_ratio/max": 2.9747977256774902,
|
|
"sampling/importance_sampling_ratio/mean": 0.5567998290061951,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9864826202392578,
|
|
"sampling/sampling_logp_difference/mean": 0.029270043596625328,
|
|
"step": 69,
|
|
"step_time": 333.99876602645963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1217447966337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2845.0,
|
|
"completions/mean_length": 905.9362182617188,
|
|
"completions/mean_terminated_length": 605.673828125,
|
|
"completions/min_length": 121.0,
|
|
"completions/min_terminated_length": 121.0,
|
|
"entropy": 0.5115884636761621,
|
|
"epoch": 1.0294117647058822,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.057038639174071336,
|
|
"learning_rate": 1.6985523143754952e-06,
|
|
"loss": -0.0316,
|
|
"num_tokens": 97092444.0,
|
|
"reward": 0.6282552480697632,
|
|
"reward_std": 0.48342809081077576,
|
|
"rewards/reward_correctness/mean": 0.6282551884651184,
|
|
"rewards/reward_correctness/std": 0.48342812061309814,
|
|
"sampling/importance_sampling_ratio/max": 2.9838263988494873,
|
|
"sampling/importance_sampling_ratio/mean": 0.5966787934303284,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.991262435913086,
|
|
"sampling/sampling_logp_difference/mean": 0.027136720716953278,
|
|
"step": 70,
|
|
"step_time": 298.9868591595441
|
|
},
|
|
{
|
|
"epoch": 1.0294117647058822,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.017857143389327184,
|
|
"eval_completions/max_length": 1204.2619047619048,
|
|
"eval_completions/max_terminated_length": 987.0714285714286,
|
|
"eval_completions/mean_length": 602.8393056960333,
|
|
"eval_completions/mean_terminated_length": 558.6147010439918,
|
|
"eval_completions/min_length": 266.26190476190476,
|
|
"eval_completions/min_terminated_length": 266.26190476190476,
|
|
"eval_entropy": 0.18525113555647077,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 97092444.0,
|
|
"eval_reward": 0.700396845738093,
|
|
"eval_reward_std": 0.4369296765043622,
|
|
"eval_rewards/reward_correctness/mean": 0.7003968279986155,
|
|
"eval_rewards/reward_correctness/std": 0.4369296765043622,
|
|
"eval_runtime": 745.652,
|
|
"eval_samples_per_second": 0.671,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.31871562657137736,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.0594283837005302,
|
|
"eval_sampling/importance_sampling_ratio/min": 3.2069904805975946e-05,
|
|
"eval_sampling/sampling_logp_difference/max": 2.500789415268671,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.045404179309982626,
|
|
"eval_steps_per_second": 0.113,
|
|
"step": 70
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.150390625,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2564.0,
|
|
"completions/mean_length": 993.2702026367188,
|
|
"completions/mean_terminated_length": 625.3110961914062,
|
|
"completions/min_length": 101.0,
|
|
"completions/min_terminated_length": 101.0,
|
|
"entropy": 0.5595204918645322,
|
|
"epoch": 1.0441176470588236,
|
|
"frac_reward_zero_std": 0.484375,
|
|
"grad_norm": 0.05220541352576947,
|
|
"learning_rate": 1.6661872078293582e-06,
|
|
"loss": -0.0331,
|
|
"num_tokens": 98757763.0,
|
|
"reward": 0.5774739980697632,
|
|
"reward_std": 0.49412214756011963,
|
|
"rewards/reward_correctness/mean": 0.5774739384651184,
|
|
"rewards/reward_correctness/std": 0.4941222071647644,
|
|
"sampling/importance_sampling_ratio/max": 2.986877202987671,
|
|
"sampling/importance_sampling_ratio/mean": 0.5434272289276123,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 2.067214012145996,
|
|
"sampling/sampling_logp_difference/mean": 0.029383253306150436,
|
|
"step": 71,
|
|
"step_time": 318.425050730817
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1705729216337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 3068.0,
|
|
"completions/mean_length": 1053.35546875,
|
|
"completions/mean_terminated_length": 638.43017578125,
|
|
"completions/min_length": 157.0,
|
|
"completions/min_terminated_length": 157.0,
|
|
"entropy": 0.6264856760390103,
|
|
"epoch": 1.0588235294117647,
|
|
"frac_reward_zero_std": 0.4609375,
|
|
"grad_norm": 0.05531152948843457,
|
|
"learning_rate": 1.6338127921706424e-06,
|
|
"loss": -0.0358,
|
|
"num_tokens": 100533745.0,
|
|
"reward": 0.5221354365348816,
|
|
"reward_std": 0.49967244267463684,
|
|
"rewards/reward_correctness/mean": 0.5221354365348816,
|
|
"rewards/reward_correctness/std": 0.49967244267463684,
|
|
"sampling/importance_sampling_ratio/max": 2.9968209266662598,
|
|
"sampling/importance_sampling_ratio/mean": 0.5030667781829834,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.4990592002868652,
|
|
"sampling/sampling_logp_difference/mean": 0.030927255749702454,
|
|
"step": 72,
|
|
"step_time": 331.996779977344
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1256510466337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 3043.0,
|
|
"completions/mean_length": 894.5814208984375,
|
|
"completions/mean_terminated_length": 582.1735229492188,
|
|
"completions/min_length": 166.0,
|
|
"completions/min_terminated_length": 166.0,
|
|
"entropy": 0.5137150924419984,
|
|
"epoch": 1.0735294117647058,
|
|
"frac_reward_zero_std": 0.515625,
|
|
"grad_norm": 0.05601516375217364,
|
|
"learning_rate": 1.6014476856245056e-06,
|
|
"loss": -0.0186,
|
|
"num_tokens": 102047766.0,
|
|
"reward": 0.6028646230697632,
|
|
"reward_std": 0.48946380615234375,
|
|
"rewards/reward_correctness/mean": 0.6028645634651184,
|
|
"rewards/reward_correctness/std": 0.48946383595466614,
|
|
"sampling/importance_sampling_ratio/max": 2.9608514308929443,
|
|
"sampling/importance_sampling_ratio/mean": 0.5782585740089417,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.042614459991455,
|
|
"sampling/sampling_logp_difference/mean": 0.028643345460295677,
|
|
"step": 73,
|
|
"step_time": 322.8676867983304
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1555989682674408,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 3022.0,
|
|
"completions/mean_length": 1002.8411865234375,
|
|
"completions/mean_terminated_length": 621.5543212890625,
|
|
"completions/min_length": 130.0,
|
|
"completions/min_terminated_length": 130.0,
|
|
"entropy": 0.5605857335031033,
|
|
"epoch": 1.088235294117647,
|
|
"frac_reward_zero_std": 0.59375,
|
|
"grad_norm": 0.04147663365501501,
|
|
"learning_rate": 1.5691105010625233e-06,
|
|
"loss": -0.0311,
|
|
"num_tokens": 103724162.0,
|
|
"reward": 0.5787760615348816,
|
|
"reward_std": 0.4939161241054535,
|
|
"rewards/reward_correctness/mean": 0.5787760615348816,
|
|
"rewards/reward_correctness/std": 0.49391618371009827,
|
|
"sampling/importance_sampling_ratio/max": 2.9887375831604004,
|
|
"sampling/importance_sampling_ratio/mean": 0.5173272490501404,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.729773759841919,
|
|
"sampling/sampling_logp_difference/mean": 0.029481105506420135,
|
|
"step": 74,
|
|
"step_time": 313.93872283026576
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12890625,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2407.0,
|
|
"completions/mean_length": 887.205078125,
|
|
"completions/mean_terminated_length": 564.169677734375,
|
|
"completions/min_length": 167.0,
|
|
"completions/min_terminated_length": 167.0,
|
|
"entropy": 0.5045698995236307,
|
|
"epoch": 1.1029411764705883,
|
|
"frac_reward_zero_std": 0.546875,
|
|
"grad_norm": 0.04361207970660597,
|
|
"learning_rate": 1.536819835298597e-06,
|
|
"loss": -0.036,
|
|
"num_tokens": 105242741.0,
|
|
"reward": 0.6302083730697632,
|
|
"reward_std": 0.4829053580760956,
|
|
"rewards/reward_correctness/mean": 0.6302083134651184,
|
|
"rewards/reward_correctness/std": 0.48290541768074036,
|
|
"sampling/importance_sampling_ratio/max": 2.95489764213562,
|
|
"sampling/importance_sampling_ratio/mean": 0.5378239750862122,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.2616276741027832,
|
|
"sampling/sampling_logp_difference/mean": 0.028588254004716873,
|
|
"step": 75,
|
|
"step_time": 324.96197805320844
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1393229216337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2865.0,
|
|
"completions/mean_length": 942.65625,
|
|
"completions/mean_terminated_length": 599.3380737304688,
|
|
"completions/min_length": 146.0,
|
|
"completions/min_terminated_length": 146.0,
|
|
"entropy": 0.5472859075525776,
|
|
"epoch": 1.1176470588235294,
|
|
"frac_reward_zero_std": 0.484375,
|
|
"grad_norm": 0.04873235501593393,
|
|
"learning_rate": 1.5045942583941002e-06,
|
|
"loss": -0.021,
|
|
"num_tokens": 106833713.0,
|
|
"reward": 0.5553385615348816,
|
|
"reward_std": 0.49709001183509827,
|
|
"rewards/reward_correctness/mean": 0.5553385615348816,
|
|
"rewards/reward_correctness/std": 0.49709007143974304,
|
|
"sampling/importance_sampling_ratio/max": 2.947646141052246,
|
|
"sampling/importance_sampling_ratio/mean": 0.5509781241416931,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 3.937028408050537,
|
|
"sampling/sampling_logp_difference/mean": 0.02943594567477703,
|
|
"step": 76,
|
|
"step_time": 328.5258017824963
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12109375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2860.0,
|
|
"completions/mean_length": 885.1627807617188,
|
|
"completions/mean_terminated_length": 583.9644165039062,
|
|
"completions/min_length": 132.0,
|
|
"completions/min_terminated_length": 132.0,
|
|
"entropy": 0.45201024040579796,
|
|
"epoch": 1.1323529411764706,
|
|
"frac_reward_zero_std": 0.5078125,
|
|
"grad_norm": 0.058636649821457894,
|
|
"learning_rate": 1.4724523029784097e-06,
|
|
"loss": -0.0379,
|
|
"num_tokens": 108318123.0,
|
|
"reward": 0.677734375,
|
|
"reward_std": 0.46749624609947205,
|
|
"rewards/reward_correctness/mean": 0.677734375,
|
|
"rewards/reward_correctness/std": 0.4674963057041168,
|
|
"sampling/importance_sampling_ratio/max": 2.9267632961273193,
|
|
"sampling/importance_sampling_ratio/mean": 0.5988159775733948,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9959211349487305,
|
|
"sampling/sampling_logp_difference/mean": 0.024872370064258575,
|
|
"step": 77,
|
|
"step_time": 344.25549465743825
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1354166716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2615.0,
|
|
"completions/mean_length": 937.7116088867188,
|
|
"completions/mean_terminated_length": 603.4254150390625,
|
|
"completions/min_length": 134.0,
|
|
"completions/min_terminated_length": 134.0,
|
|
"entropy": 0.5290156111586839,
|
|
"epoch": 1.1470588235294117,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.056322046975264516,
|
|
"learning_rate": 1.4404124535909613e-06,
|
|
"loss": -0.0319,
|
|
"num_tokens": 109893544.0,
|
|
"reward": 0.6536458730697632,
|
|
"reward_std": 0.4759626090526581,
|
|
"rewards/reward_correctness/mean": 0.6536458134651184,
|
|
"rewards/reward_correctness/std": 0.47596263885498047,
|
|
"sampling/importance_sampling_ratio/max": 2.9960668087005615,
|
|
"sampling/importance_sampling_ratio/mean": 0.573235273361206,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.4308843612670898,
|
|
"sampling/sampling_logp_difference/mean": 0.02814130112528801,
|
|
"step": 78,
|
|
"step_time": 305.98608493292704
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1393229216337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2859.0,
|
|
"completions/mean_length": 959.6171875,
|
|
"completions/mean_terminated_length": 617.6732177734375,
|
|
"completions/min_length": 118.0,
|
|
"completions/min_terminated_length": 118.0,
|
|
"entropy": 0.5062569079454988,
|
|
"epoch": 1.161764705882353,
|
|
"frac_reward_zero_std": 0.4609375,
|
|
"grad_norm": 0.04613175848104597,
|
|
"learning_rate": 1.4084931360509656e-06,
|
|
"loss": -0.0304,
|
|
"num_tokens": 111509776.0,
|
|
"reward": 0.6471354365348816,
|
|
"reward_std": 0.4780166447162628,
|
|
"rewards/reward_correctness/mean": 0.6471354365348816,
|
|
"rewards/reward_correctness/std": 0.4780166745185852,
|
|
"sampling/importance_sampling_ratio/max": 2.997708320617676,
|
|
"sampling/importance_sampling_ratio/mean": 0.5594146251678467,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 2.120018482208252,
|
|
"sampling/sampling_logp_difference/mean": 0.027207151055336,
|
|
"step": 79,
|
|
"step_time": 325.19561221823096
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1471354216337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2118.0,
|
|
"completions/mean_length": 977.91015625,
|
|
"completions/mean_terminated_length": 616.6397094726562,
|
|
"completions/min_length": 174.0,
|
|
"completions/min_terminated_length": 174.0,
|
|
"entropy": 0.5284926120657474,
|
|
"epoch": 1.1764705882352942,
|
|
"frac_reward_zero_std": 0.5,
|
|
"grad_norm": 0.04659602550832931,
|
|
"learning_rate": 1.376712706860888e-06,
|
|
"loss": -0.0295,
|
|
"num_tokens": 113157622.0,
|
|
"reward": 0.5631510615348816,
|
|
"reward_std": 0.4961574077606201,
|
|
"rewards/reward_correctness/mean": 0.5631510615348816,
|
|
"rewards/reward_correctness/std": 0.4961574673652649,
|
|
"sampling/importance_sampling_ratio/max": 2.9972426891326904,
|
|
"sampling/importance_sampling_ratio/mean": 0.5413715243339539,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.0071921348571777,
|
|
"sampling/sampling_logp_difference/mean": 0.028690865263342857,
|
|
"step": 80,
|
|
"step_time": 326.1782124824822
|
|
},
|
|
{
|
|
"epoch": 1.1764705882352942,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.015873016346068608,
|
|
"eval_completions/max_length": 1213.75,
|
|
"eval_completions/max_terminated_length": 1036.107142857143,
|
|
"eval_completions/mean_length": 619.6428756713867,
|
|
"eval_completions/mean_terminated_length": 581.0579517909458,
|
|
"eval_completions/min_length": 269.5833333333333,
|
|
"eval_completions/min_terminated_length": 269.5833333333333,
|
|
"eval_entropy": 0.17011943459510803,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 113157622.0,
|
|
"eval_reward": 0.7182539861117091,
|
|
"eval_reward_std": 0.4072274203811373,
|
|
"eval_rewards/reward_correctness/mean": 0.7182539705009687,
|
|
"eval_rewards/reward_correctness/std": 0.4072274182524,
|
|
"eval_runtime": 748.4449,
|
|
"eval_samples_per_second": 0.668,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.2644888570031894,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.05727769766408961,
|
|
"eval_sampling/importance_sampling_ratio/min": 4.0693722128255106e-05,
|
|
"eval_sampling/sampling_logp_difference/max": 2.371966535136813,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.042117656696410405,
|
|
"eval_steps_per_second": 0.112,
|
|
"step": 80
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1529947966337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2791.0,
|
|
"completions/mean_length": 995.4479370117188,
|
|
"completions/mean_terminated_length": 620.3597412109375,
|
|
"completions/min_length": 131.0,
|
|
"completions/min_terminated_length": 131.0,
|
|
"entropy": 0.5432424978353083,
|
|
"epoch": 1.1911764705882353,
|
|
"frac_reward_zero_std": 0.4375,
|
|
"grad_norm": 0.05378785526390247,
|
|
"learning_rate": 1.3450894426497986e-06,
|
|
"loss": -0.0294,
|
|
"num_tokens": 114818414.0,
|
|
"reward": 0.5611979365348816,
|
|
"reward_std": 0.4964022934436798,
|
|
"rewards/reward_correctness/mean": 0.5611979365348816,
|
|
"rewards/reward_correctness/std": 0.4964022934436798,
|
|
"sampling/importance_sampling_ratio/max": 2.973362684249878,
|
|
"sampling/importance_sampling_ratio/mean": 0.5383284091949463,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.353340744972229,
|
|
"sampling/sampling_logp_difference/mean": 0.028549566864967346,
|
|
"step": 81,
|
|
"step_time": 336.11909542698413
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1432291716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2812.0,
|
|
"completions/mean_length": 943.6744995117188,
|
|
"completions/mean_terminated_length": 587.8753662109375,
|
|
"completions/min_length": 157.0,
|
|
"completions/min_terminated_length": 157.0,
|
|
"entropy": 0.5193857927224599,
|
|
"epoch": 1.2058823529411764,
|
|
"frac_reward_zero_std": 0.4609375,
|
|
"grad_norm": 0.056012673442915696,
|
|
"learning_rate": 1.313641529662647e-06,
|
|
"loss": -0.043,
|
|
"num_tokens": 116402598.0,
|
|
"reward": 0.64453125,
|
|
"reward_std": 0.47881096601486206,
|
|
"rewards/reward_correctness/mean": 0.64453125,
|
|
"rewards/reward_correctness/std": 0.47881099581718445,
|
|
"sampling/importance_sampling_ratio/max": 2.9883127212524414,
|
|
"sampling/importance_sampling_ratio/mean": 0.5390397906303406,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.1875901222229004,
|
|
"sampling/sampling_logp_difference/mean": 0.029155386611819267,
|
|
"step": 82,
|
|
"step_time": 328.2109436998144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1145833358168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2260.0,
|
|
"completions/mean_length": 880.8619995117188,
|
|
"completions/mean_terminated_length": 597.6353149414062,
|
|
"completions/min_length": 166.0,
|
|
"completions/min_terminated_length": 166.0,
|
|
"entropy": 0.4548957828665152,
|
|
"epoch": 1.2205882352941178,
|
|
"frac_reward_zero_std": 0.5703125,
|
|
"grad_norm": 0.04006469425367404,
|
|
"learning_rate": 1.2823870533015295e-06,
|
|
"loss": -0.0327,
|
|
"num_tokens": 117905218.0,
|
|
"reward": 0.6380208730697632,
|
|
"reward_std": 0.4807293117046356,
|
|
"rewards/reward_correctness/mean": 0.6380208134651184,
|
|
"rewards/reward_correctness/std": 0.480729341506958,
|
|
"sampling/importance_sampling_ratio/max": 2.9866812229156494,
|
|
"sampling/importance_sampling_ratio/mean": 0.5929659008979797,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.315820574760437,
|
|
"sampling/sampling_logp_difference/mean": 0.02623382955789566,
|
|
"step": 83,
|
|
"step_time": 327.8097211890854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.15234375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2345.0,
|
|
"completions/mean_length": 979.0045776367188,
|
|
"completions/mean_terminated_length": 603.8709716796875,
|
|
"completions/min_length": 113.0,
|
|
"completions/min_terminated_length": 113.0,
|
|
"entropy": 0.5572732011787593,
|
|
"epoch": 1.2352941176470589,
|
|
"frac_reward_zero_std": 0.5625,
|
|
"grad_norm": 0.04990192862275496,
|
|
"learning_rate": 1.2513439877249363e-06,
|
|
"loss": -0.0493,
|
|
"num_tokens": 119565737.0,
|
|
"reward": 0.6236979365348816,
|
|
"reward_std": 0.4846149981021881,
|
|
"rewards/reward_correctness/mean": 0.6236979365348816,
|
|
"rewards/reward_correctness/std": 0.4846150279045105,
|
|
"sampling/importance_sampling_ratio/max": 2.9869635105133057,
|
|
"sampling/importance_sampling_ratio/mean": 0.5159881114959717,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.264847993850708,
|
|
"sampling/sampling_logp_difference/mean": 0.030388643965125084,
|
|
"step": 84,
|
|
"step_time": 325.7631025062874
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.162109375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2351.0,
|
|
"completions/mean_length": 979.5514526367188,
|
|
"completions/mean_terminated_length": 574.71875,
|
|
"completions/min_length": 136.0,
|
|
"completions/min_terminated_length": 136.0,
|
|
"entropy": 0.5549764286261052,
|
|
"epoch": 1.25,
|
|
"frac_reward_zero_std": 0.53125,
|
|
"grad_norm": 0.05425624294733418,
|
|
"learning_rate": 1.220530185510985e-06,
|
|
"loss": -0.0292,
|
|
"num_tokens": 121217736.0,
|
|
"reward": 0.6087239980697632,
|
|
"reward_std": 0.48819488286972046,
|
|
"rewards/reward_correctness/mean": 0.6087239384651184,
|
|
"rewards/reward_correctness/std": 0.48819488286972046,
|
|
"sampling/importance_sampling_ratio/max": 2.8903768062591553,
|
|
"sampling/importance_sampling_ratio/mean": 0.5493928790092468,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 2.14804744720459,
|
|
"sampling/sampling_logp_difference/mean": 0.029596565291285515,
|
|
"step": 85,
|
|
"step_time": 331.2769885966554
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1197916716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2747.0,
|
|
"completions/mean_length": 938.671875,
|
|
"completions/mean_terminated_length": 648.3372802734375,
|
|
"completions/min_length": 152.0,
|
|
"completions/min_terminated_length": 152.0,
|
|
"entropy": 0.46246786625124514,
|
|
"epoch": 1.2647058823529411,
|
|
"frac_reward_zero_std": 0.515625,
|
|
"grad_norm": 0.049832296894574454,
|
|
"learning_rate": 1.189963367390565e-06,
|
|
"loss": -0.0236,
|
|
"num_tokens": 122787444.0,
|
|
"reward": 0.6236979365348816,
|
|
"reward_std": 0.4846149981021881,
|
|
"rewards/reward_correctness/mean": 0.6236979365348816,
|
|
"rewards/reward_correctness/std": 0.4846150279045105,
|
|
"sampling/importance_sampling_ratio/max": 2.9678292274475098,
|
|
"sampling/importance_sampling_ratio/mean": 0.5417976379394531,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.1519529819488525,
|
|
"sampling/sampling_logp_difference/mean": 0.02557907998561859,
|
|
"step": 86,
|
|
"step_time": 307.65380067098886
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.13671875,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2030.0,
|
|
"completions/mean_length": 986.380859375,
|
|
"completions/mean_terminated_length": 656.8952026367188,
|
|
"completions/min_length": 165.0,
|
|
"completions/min_terminated_length": 165.0,
|
|
"entropy": 0.49913884582929313,
|
|
"epoch": 1.2794117647058822,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.05934246184930205,
|
|
"learning_rate": 1.159661112056314e-06,
|
|
"loss": -0.0092,
|
|
"num_tokens": 124479045.0,
|
|
"reward": 0.5924479365348816,
|
|
"reward_std": 0.4915390610694885,
|
|
"rewards/reward_correctness/mean": 0.5924479365348816,
|
|
"rewards/reward_correctness/std": 0.4915390610694885,
|
|
"sampling/importance_sampling_ratio/max": 2.974931240081787,
|
|
"sampling/importance_sampling_ratio/mean": 0.46693944931030273,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 27.75,
|
|
"sampling/sampling_logp_difference/mean": 0.043718449771404266,
|
|
"step": 87,
|
|
"step_time": 343.7939372379333
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0891927108168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1973.0,
|
|
"completions/mean_length": 822.8307495117188,
|
|
"completions/mean_terminated_length": 602.7734375,
|
|
"completions/min_length": 182.0,
|
|
"completions/min_terminated_length": 182.0,
|
|
"entropy": 0.40294282842660323,
|
|
"epoch": 1.2941176470588236,
|
|
"frac_reward_zero_std": 0.5078125,
|
|
"grad_norm": 0.057870776481504994,
|
|
"learning_rate": 1.1296408460532715e-06,
|
|
"loss": -0.0334,
|
|
"num_tokens": 125892913.0,
|
|
"reward": 0.6725260615348816,
|
|
"reward_std": 0.4694445729255676,
|
|
"rewards/reward_correctness/mean": 0.6725260615348816,
|
|
"rewards/reward_correctness/std": 0.4694446325302124,
|
|
"sampling/importance_sampling_ratio/max": 2.993168830871582,
|
|
"sampling/importance_sampling_ratio/mean": 0.6000463366508484,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.2202494144439697,
|
|
"sampling/sampling_logp_difference/mean": 0.023823007941246033,
|
|
"step": 88,
|
|
"step_time": 304.67693279404193
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.107421875,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2732.0,
|
|
"completions/mean_length": 899.1484375,
|
|
"completions/mean_terminated_length": 637.6455078125,
|
|
"completions/min_length": 145.0,
|
|
"completions/min_terminated_length": 145.0,
|
|
"entropy": 0.4273973684757948,
|
|
"epoch": 1.3088235294117647,
|
|
"frac_reward_zero_std": 0.53125,
|
|
"grad_norm": 0.05036942737826866,
|
|
"learning_rate": 1.0999198337570392e-06,
|
|
"loss": -0.0112,
|
|
"num_tokens": 127410409.0,
|
|
"reward": 0.6126302480697632,
|
|
"reward_std": 0.48730790615081787,
|
|
"rewards/reward_correctness/mean": 0.6126301884651184,
|
|
"rewards/reward_correctness/std": 0.48730793595314026,
|
|
"sampling/importance_sampling_ratio/max": 2.9690043926239014,
|
|
"sampling/importance_sampling_ratio/mean": 0.5925201773643494,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9818124771118164,
|
|
"sampling/sampling_logp_difference/mean": 0.02451782487332821,
|
|
"step": 89,
|
|
"step_time": 298.9374023540877
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1223958358168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2859.0,
|
|
"completions/mean_length": 954.78125,
|
|
"completions/mean_terminated_length": 659.6676635742188,
|
|
"completions/min_length": 175.0,
|
|
"completions/min_terminated_length": 175.0,
|
|
"entropy": 0.43672167870681733,
|
|
"epoch": 1.3235294117647058,
|
|
"frac_reward_zero_std": 0.5234375,
|
|
"grad_norm": 0.043707152943409704,
|
|
"learning_rate": 1.0705151674451938e-06,
|
|
"loss": -0.0373,
|
|
"num_tokens": 129020677.0,
|
|
"reward": 0.595703125,
|
|
"reward_std": 0.4909152686595917,
|
|
"rewards/reward_correctness/mean": 0.595703125,
|
|
"rewards/reward_correctness/std": 0.49091529846191406,
|
|
"sampling/importance_sampling_ratio/max": 2.9962658882141113,
|
|
"sampling/importance_sampling_ratio/mean": 0.5464567542076111,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.6809594631195068,
|
|
"sampling/sampling_logp_difference/mean": 0.02434036135673523,
|
|
"step": 90,
|
|
"step_time": 322.3661844804883
|
|
},
|
|
{
|
|
"epoch": 1.3235294117647058,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.013888889302810034,
|
|
"eval_completions/max_length": 1180.452380952381,
|
|
"eval_completions/max_terminated_length": 1016.547619047619,
|
|
"eval_completions/mean_length": 611.1190646943592,
|
|
"eval_completions/mean_terminated_length": 576.907953171503,
|
|
"eval_completions/min_length": 273.5952380952381,
|
|
"eval_completions/min_terminated_length": 273.5952380952381,
|
|
"eval_entropy": 0.1594409356337218,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 129020677.0,
|
|
"eval_reward": 0.72420636706409,
|
|
"eval_reward_std": 0.4057711760203044,
|
|
"eval_rewards/reward_correctness/mean": 0.7242063500341915,
|
|
"eval_rewards/reward_correctness/std": 0.405771173891567,
|
|
"eval_runtime": 727.5572,
|
|
"eval_samples_per_second": 0.687,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.2706919278598867,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.05651876312922035,
|
|
"eval_sampling/importance_sampling_ratio/min": 3.644251580033999e-05,
|
|
"eval_sampling/sampling_logp_difference/max": 2.489054736636934,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.03981469148060396,
|
|
"eval_steps_per_second": 0.115,
|
|
"step": 90
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1184895858168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2849.0,
|
|
"completions/mean_length": 886.9453125,
|
|
"completions/mean_terminated_length": 594.0214233398438,
|
|
"completions/min_length": 162.0,
|
|
"completions/min_terminated_length": 162.0,
|
|
"entropy": 0.4434245915035717,
|
|
"epoch": 1.3382352941176472,
|
|
"frac_reward_zero_std": 0.53125,
|
|
"grad_norm": 0.049403019001355145,
|
|
"learning_rate": 1.0414437574676832e-06,
|
|
"loss": -0.0285,
|
|
"num_tokens": 130553533.0,
|
|
"reward": 0.6666666865348816,
|
|
"reward_std": 0.4715580344200134,
|
|
"rewards/reward_correctness/mean": 0.6666666865348816,
|
|
"rewards/reward_correctness/std": 0.4715580344200134,
|
|
"sampling/importance_sampling_ratio/max": 2.987356662750244,
|
|
"sampling/importance_sampling_ratio/mean": 0.4857485294342041,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 23.5,
|
|
"sampling/sampling_logp_difference/mean": 0.0383615642786026,
|
|
"step": 91,
|
|
"step_time": 357.55924094608054
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1276041716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2910.0,
|
|
"completions/mean_length": 936.6178588867188,
|
|
"completions/mean_terminated_length": 624.3798828125,
|
|
"completions/min_length": 146.0,
|
|
"completions/min_terminated_length": 146.0,
|
|
"entropy": 0.4767748566227965,
|
|
"epoch": 1.3529411764705883,
|
|
"frac_reward_zero_std": 0.5078125,
|
|
"grad_norm": 0.047660426114125534,
|
|
"learning_rate": 1.0127223225218379e-06,
|
|
"loss": -0.0261,
|
|
"num_tokens": 132141518.0,
|
|
"reward": 0.6321614980697632,
|
|
"reward_std": 0.4823741316795349,
|
|
"rewards/reward_correctness/mean": 0.6321614384651184,
|
|
"rewards/reward_correctness/std": 0.4823741614818573,
|
|
"sampling/importance_sampling_ratio/max": 2.9987378120422363,
|
|
"sampling/importance_sampling_ratio/mean": 0.5875497460365295,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9881277084350586,
|
|
"sampling/sampling_logp_difference/mean": 0.026067327708005905,
|
|
"step": 92,
|
|
"step_time": 323.0196398496628
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.103515625,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2182.0,
|
|
"completions/mean_length": 850.3541870117188,
|
|
"completions/mean_terminated_length": 594.8068237304688,
|
|
"completions/min_length": 113.0,
|
|
"completions/min_terminated_length": 113.0,
|
|
"entropy": 0.41502448660321534,
|
|
"epoch": 1.3676470588235294,
|
|
"frac_reward_zero_std": 0.5390625,
|
|
"grad_norm": 0.047670989020446516,
|
|
"learning_rate": 9.843673800376037e-07,
|
|
"loss": -0.0073,
|
|
"num_tokens": 133603158.0,
|
|
"reward": 0.6373698115348816,
|
|
"reward_std": 0.480915904045105,
|
|
"rewards/reward_correctness/mean": 0.6373698115348816,
|
|
"rewards/reward_correctness/std": 0.480915904045105,
|
|
"sampling/importance_sampling_ratio/max": 2.926886558532715,
|
|
"sampling/importance_sampling_ratio/mean": 0.5859780311584473,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.733330488204956,
|
|
"sampling/sampling_logp_difference/mean": 0.02443034015595913,
|
|
"step": 93,
|
|
"step_time": 321.0396940037608
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.095703125,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2675.0,
|
|
"completions/mean_length": 832.0885620117188,
|
|
"completions/mean_terminated_length": 595.0352783203125,
|
|
"completions/min_length": 120.0,
|
|
"completions/min_terminated_length": 120.0,
|
|
"entropy": 0.4050459030549973,
|
|
"epoch": 1.3823529411764706,
|
|
"frac_reward_zero_std": 0.5625,
|
|
"grad_norm": 0.05508775780210909,
|
|
"learning_rate": 9.563952366785246e-07,
|
|
"loss": -0.0288,
|
|
"num_tokens": 135027286.0,
|
|
"reward": 0.6360677480697632,
|
|
"reward_std": 0.4812861680984497,
|
|
"rewards/reward_correctness/mean": 0.6360676884651184,
|
|
"rewards/reward_correctness/std": 0.4812861680984497,
|
|
"sampling/importance_sampling_ratio/max": 2.9579274654388428,
|
|
"sampling/importance_sampling_ratio/mean": 0.5902597904205322,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.4866766929626465,
|
|
"sampling/sampling_logp_difference/mean": 0.024181541055440903,
|
|
"step": 94,
|
|
"step_time": 315.43966129794717
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1243489608168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2420.0,
|
|
"completions/mean_length": 941.1107177734375,
|
|
"completions/mean_terminated_length": 638.5300903320312,
|
|
"completions/min_length": 129.0,
|
|
"completions/min_terminated_length": 129.0,
|
|
"entropy": 0.46013247850351036,
|
|
"epoch": 1.3970588235294117,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.04858255336422534,
|
|
"learning_rate": 9.288219789639276e-07,
|
|
"loss": -0.0079,
|
|
"num_tokens": 136612824.0,
|
|
"reward": 0.6061198115348816,
|
|
"reward_std": 0.48876795172691345,
|
|
"rewards/reward_correctness/mean": 0.6061198115348816,
|
|
"rewards/reward_correctness/std": 0.48876795172691345,
|
|
"sampling/importance_sampling_ratio/max": 2.995532989501953,
|
|
"sampling/importance_sampling_ratio/mean": 0.5338165163993835,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 3.132884979248047,
|
|
"sampling/sampling_logp_difference/mean": 0.026240212842822075,
|
|
"step": 95,
|
|
"step_time": 320.3804359170608
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1126302108168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2845.0,
|
|
"completions/mean_length": 899.4466552734375,
|
|
"completions/mean_terminated_length": 623.693359375,
|
|
"completions/min_length": 149.0,
|
|
"completions/min_terminated_length": 149.0,
|
|
"entropy": 0.4312205637106672,
|
|
"epoch": 1.4117647058823528,
|
|
"frac_reward_zero_std": 0.515625,
|
|
"grad_norm": 0.048290932446018155,
|
|
"learning_rate": 9.016634640177203e-07,
|
|
"loss": -0.0346,
|
|
"num_tokens": 138131150.0,
|
|
"reward": 0.6145833730697632,
|
|
"reward_std": 0.48685210943222046,
|
|
"rewards/reward_correctness/mean": 0.6145833134651184,
|
|
"rewards/reward_correctness/std": 0.48685210943222046,
|
|
"sampling/importance_sampling_ratio/max": 2.995443820953369,
|
|
"sampling/importance_sampling_ratio/mean": 0.583075225353241,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.4211063385009766,
|
|
"sampling/sampling_logp_difference/mean": 0.025148890912532806,
|
|
"step": 96,
|
|
"step_time": 315.7374519892037
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09765625,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2736.0,
|
|
"completions/mean_length": 847.4447021484375,
|
|
"completions/mean_terminated_length": 606.69189453125,
|
|
"completions/min_length": 120.0,
|
|
"completions/min_terminated_length": 120.0,
|
|
"entropy": 0.395701271481812,
|
|
"epoch": 1.4264705882352942,
|
|
"frac_reward_zero_std": 0.5625,
|
|
"grad_norm": 0.04319162219933892,
|
|
"learning_rate": 8.74935310449101e-07,
|
|
"loss": -0.019,
|
|
"num_tokens": 139587937.0,
|
|
"reward": 0.6692708730697632,
|
|
"reward_std": 0.47062891721725464,
|
|
"rewards/reward_correctness/mean": 0.6692708134651184,
|
|
"rewards/reward_correctness/std": 0.47062891721725464,
|
|
"sampling/importance_sampling_ratio/max": 2.9770994186401367,
|
|
"sampling/importance_sampling_ratio/mean": 0.5773332715034485,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9985542297363281,
|
|
"sampling/sampling_logp_difference/mean": 0.023664306849241257,
|
|
"step": 97,
|
|
"step_time": 329.263949975837
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1100260466337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2920.0,
|
|
"completions/mean_length": 890.892578125,
|
|
"completions/mean_terminated_length": 621.432373046875,
|
|
"completions/min_length": 181.0,
|
|
"completions/min_terminated_length": 181.0,
|
|
"entropy": 0.41403548134258017,
|
|
"epoch": 1.4411764705882353,
|
|
"frac_reward_zero_std": 0.5703125,
|
|
"grad_norm": 0.05443612081724022,
|
|
"learning_rate": 8.486528893704481e-07,
|
|
"loss": -0.0084,
|
|
"num_tokens": 141091804.0,
|
|
"reward": 0.6516927480697632,
|
|
"reward_std": 0.4765890836715698,
|
|
"rewards/reward_correctness/mean": 0.6516926884651184,
|
|
"rewards/reward_correctness/std": 0.4765891432762146,
|
|
"sampling/importance_sampling_ratio/max": 2.99269700050354,
|
|
"sampling/importance_sampling_ratio/mean": 0.5741823315620422,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 3.037632465362549,
|
|
"sampling/sampling_logp_difference/mean": 0.02437897026538849,
|
|
"step": 98,
|
|
"step_time": 318.34623634815216
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0924479216337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2389.0,
|
|
"completions/mean_length": 816.14453125,
|
|
"completions/mean_terminated_length": 586.3515014648438,
|
|
"completions/min_length": 107.0,
|
|
"completions/min_terminated_length": 107.0,
|
|
"entropy": 0.3746476907399483,
|
|
"epoch": 1.4558823529411764,
|
|
"frac_reward_zero_std": 0.546875,
|
|
"grad_norm": 0.05116238263191225,
|
|
"learning_rate": 8.228313155575304e-07,
|
|
"loss": -0.0001,
|
|
"num_tokens": 142478098.0,
|
|
"reward": 0.6341146230697632,
|
|
"reward_std": 0.48183441162109375,
|
|
"rewards/reward_correctness/mean": 0.6341145634651184,
|
|
"rewards/reward_correctness/std": 0.48183444142341614,
|
|
"sampling/importance_sampling_ratio/max": 2.990213632583618,
|
|
"sampling/importance_sampling_ratio/mean": 0.623569905757904,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.2119526863098145,
|
|
"sampling/sampling_logp_difference/mean": 0.023311734199523926,
|
|
"step": 99,
|
|
"step_time": 293.37062060274184
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1276041716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 3069.0,
|
|
"completions/mean_length": 935.8157958984375,
|
|
"completions/mean_terminated_length": 623.3589477539062,
|
|
"completions/min_length": 163.0,
|
|
"completions/min_terminated_length": 163.0,
|
|
"entropy": 0.46815127378795296,
|
|
"epoch": 1.4705882352941178,
|
|
"frac_reward_zero_std": 0.4375,
|
|
"grad_norm": 0.06215867891136184,
|
|
"learning_rate": 7.97485438757144e-07,
|
|
"loss": -0.0102,
|
|
"num_tokens": 144058587.0,
|
|
"reward": 0.607421875,
|
|
"reward_std": 0.4884832203388214,
|
|
"rewards/reward_correctness/mean": 0.607421875,
|
|
"rewards/reward_correctness/std": 0.4884832501411438,
|
|
"sampling/importance_sampling_ratio/max": 2.9991674423217773,
|
|
"sampling/importance_sampling_ratio/mean": 0.5880123972892761,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 2.526287078857422,
|
|
"sampling/sampling_logp_difference/mean": 0.026227515190839767,
|
|
"step": 100,
|
|
"step_time": 321.7099755220115
|
|
},
|
|
{
|
|
"epoch": 1.4705882352941178,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.027777778605620067,
|
|
"eval_completions/max_length": 1247.7380952380952,
|
|
"eval_completions/max_terminated_length": 973.8333333333334,
|
|
"eval_completions/mean_length": 641.6131166730609,
|
|
"eval_completions/mean_terminated_length": 572.130375453404,
|
|
"eval_completions/min_length": 266.4761904761905,
|
|
"eval_completions/min_terminated_length": 266.4761904761905,
|
|
"eval_entropy": 0.14853831895050548,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 144058587.0,
|
|
"eval_reward": 0.7301587504999978,
|
|
"eval_reward_std": 0.4032438102932203,
|
|
"eval_rewards/reward_correctness/mean": 0.7301587299222038,
|
|
"eval_rewards/reward_correctness/std": 0.4032438120671681,
|
|
"eval_runtime": 775.7291,
|
|
"eval_samples_per_second": 0.645,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.35592690231056795,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.07854402060807583,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.00017360338998916516,
|
|
"eval_sampling/sampling_logp_difference/max": 2.4407017798650834,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.03718282767970647,
|
|
"eval_steps_per_second": 0.108,
|
|
"step": 100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.095703125,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2770.0,
|
|
"completions/mean_length": 844.6907958984375,
|
|
"completions/mean_terminated_length": 609.3268432617188,
|
|
"completions/min_length": 164.0,
|
|
"completions/min_terminated_length": 164.0,
|
|
"entropy": 0.39180235320236534,
|
|
"epoch": 1.4852941176470589,
|
|
"frac_reward_zero_std": 0.5234375,
|
|
"grad_norm": 0.06088990844615045,
|
|
"learning_rate": 7.726298351471607e-07,
|
|
"loss": -0.0316,
|
|
"num_tokens": 145506620.0,
|
|
"reward": 0.603515625,
|
|
"reward_std": 0.48932644724845886,
|
|
"rewards/reward_correctness/mean": 0.603515625,
|
|
"rewards/reward_correctness/std": 0.48932647705078125,
|
|
"sampling/importance_sampling_ratio/max": 2.9502859115600586,
|
|
"sampling/importance_sampling_ratio/mean": 0.5787551999092102,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.7813924551010132,
|
|
"sampling/sampling_logp_difference/mean": 0.02406127005815506,
|
|
"step": 101,
|
|
"step_time": 315.4343173727393
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1165364608168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2674.0,
|
|
"completions/mean_length": 902.7318115234375,
|
|
"completions/mean_terminated_length": 617.4561767578125,
|
|
"completions/min_length": 159.0,
|
|
"completions/min_terminated_length": 159.0,
|
|
"entropy": 0.4538170258747414,
|
|
"epoch": 1.5,
|
|
"frac_reward_zero_std": 0.5546875,
|
|
"grad_norm": 0.04069827271561709,
|
|
"learning_rate": 7.482787989539021e-07,
|
|
"loss": -0.0336,
|
|
"num_tokens": 147049660.0,
|
|
"reward": 0.6106771230697632,
|
|
"reward_std": 0.48775550723075867,
|
|
"rewards/reward_correctness/mean": 0.6106770634651184,
|
|
"rewards/reward_correctness/std": 0.48775553703308105,
|
|
"sampling/importance_sampling_ratio/max": 2.9973056316375732,
|
|
"sampling/importance_sampling_ratio/mean": 0.5489984154701233,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 3.696782350540161,
|
|
"sampling/sampling_logp_difference/mean": 0.026487788185477257,
|
|
"step": 102,
|
|
"step_time": 315.7598335049115
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.095703125,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2178.0,
|
|
"completions/mean_length": 836.509765625,
|
|
"completions/mean_terminated_length": 599.9243774414062,
|
|
"completions/min_length": 129.0,
|
|
"completions/min_terminated_length": 129.0,
|
|
"entropy": 0.38416015822440386,
|
|
"epoch": 1.5147058823529411,
|
|
"frac_reward_zero_std": 0.546875,
|
|
"grad_norm": 0.0751106349162355,
|
|
"learning_rate": 7.244463342316648e-07,
|
|
"loss": -0.0074,
|
|
"num_tokens": 148463419.0,
|
|
"reward": 0.6458333730697632,
|
|
"reward_std": 0.47841572761535645,
|
|
"rewards/reward_correctness/mean": 0.6458333134651184,
|
|
"rewards/reward_correctness/std": 0.4784157872200012,
|
|
"sampling/importance_sampling_ratio/max": 2.998680353164673,
|
|
"sampling/importance_sampling_ratio/mean": 0.6221364736557007,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.7048001289367676,
|
|
"sampling/sampling_logp_difference/mean": 0.02271461859345436,
|
|
"step": 103,
|
|
"step_time": 303.2456877552904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0924479216337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1976.0,
|
|
"completions/mean_length": 850.232421875,
|
|
"completions/mean_terminated_length": 624.4139404296875,
|
|
"completions/min_length": 158.0,
|
|
"completions/min_terminated_length": 158.0,
|
|
"entropy": 0.3963266082573682,
|
|
"epoch": 1.5294117647058822,
|
|
"frac_reward_zero_std": 0.5234375,
|
|
"grad_norm": 0.04814880690491371,
|
|
"learning_rate": 7.011461468091183e-07,
|
|
"loss": -0.03,
|
|
"num_tokens": 149915332.0,
|
|
"reward": 0.603515625,
|
|
"reward_std": 0.48932647705078125,
|
|
"rewards/reward_correctness/mean": 0.603515625,
|
|
"rewards/reward_correctness/std": 0.48932647705078125,
|
|
"sampling/importance_sampling_ratio/max": 2.9855146408081055,
|
|
"sampling/importance_sampling_ratio/mean": 0.5602810978889465,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 3.655331611633301,
|
|
"sampling/sampling_logp_difference/mean": 0.023774094879627228,
|
|
"step": 104,
|
|
"step_time": 311.6580576309934
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0944010466337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2853.0,
|
|
"completions/mean_length": 838.3307495117188,
|
|
"completions/mean_terminated_length": 605.4895629882812,
|
|
"completions/min_length": 155.0,
|
|
"completions/min_terminated_length": 155.0,
|
|
"entropy": 0.3826165738864802,
|
|
"epoch": 1.5441176470588234,
|
|
"frac_reward_zero_std": 0.484375,
|
|
"grad_norm": 0.05765692003692507,
|
|
"learning_rate": 6.783916364072101e-07,
|
|
"loss": -0.0055,
|
|
"num_tokens": 151347980.0,
|
|
"reward": 0.6145833730697632,
|
|
"reward_std": 0.48685207962989807,
|
|
"rewards/reward_correctness/mean": 0.6145833134651184,
|
|
"rewards/reward_correctness/std": 0.48685210943222046,
|
|
"sampling/importance_sampling_ratio/max": 2.982243061065674,
|
|
"sampling/importance_sampling_ratio/mean": 0.5794587731361389,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.5441890954971313,
|
|
"sampling/sampling_logp_difference/mean": 0.02304181084036827,
|
|
"step": 105,
|
|
"step_time": 293.3862261297181
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0787760466337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2716.0,
|
|
"completions/mean_length": 823.7578125,
|
|
"completions/mean_terminated_length": 631.5053100585938,
|
|
"completions/min_length": 130.0,
|
|
"completions/min_terminated_length": 130.0,
|
|
"entropy": 0.3542632268508896,
|
|
"epoch": 1.5588235294117647,
|
|
"frac_reward_zero_std": 0.53125,
|
|
"grad_norm": 0.0638999919867717,
|
|
"learning_rate": 6.561958889331121e-07,
|
|
"loss": -0.0204,
|
|
"num_tokens": 152743628.0,
|
|
"reward": 0.6484375,
|
|
"reward_std": 0.4776136577129364,
|
|
"rewards/reward_correctness/mean": 0.6484375,
|
|
"rewards/reward_correctness/std": 0.4776136875152588,
|
|
"sampling/importance_sampling_ratio/max": 2.961015462875366,
|
|
"sampling/importance_sampling_ratio/mean": 0.6033360958099365,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.9093666076660156,
|
|
"sampling/sampling_logp_difference/mean": 0.021682944148778915,
|
|
"step": 106,
|
|
"step_time": 285.9305932777934
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.091796875,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2778.0,
|
|
"completions/mean_length": 837.1588745117188,
|
|
"completions/mean_terminated_length": 614.7268676757812,
|
|
"completions/min_length": 132.0,
|
|
"completions/min_terminated_length": 132.0,
|
|
"entropy": 0.4021087958244607,
|
|
"epoch": 1.5735294117647058,
|
|
"frac_reward_zero_std": 0.5390625,
|
|
"grad_norm": 0.04555968292142207,
|
|
"learning_rate": 6.345716689546361e-07,
|
|
"loss": -0.0165,
|
|
"num_tokens": 154186080.0,
|
|
"reward": 0.6276041865348816,
|
|
"reward_std": 0.4836004674434662,
|
|
"rewards/reward_correctness/mean": 0.6276041865348816,
|
|
"rewards/reward_correctness/std": 0.4836004674434662,
|
|
"sampling/importance_sampling_ratio/max": 2.9571053981781006,
|
|
"sampling/importance_sampling_ratio/mean": 0.4639313220977783,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 22.87156105041504,
|
|
"sampling/sampling_logp_difference/mean": 0.04087618738412857,
|
|
"step": 107,
|
|
"step_time": 334.3754716110416
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1145833358168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2424.0,
|
|
"completions/mean_length": 895.3118896484375,
|
|
"completions/mean_terminated_length": 614.1749877929688,
|
|
"completions/min_length": 154.0,
|
|
"completions/min_terminated_length": 154.0,
|
|
"entropy": 0.41992489667609334,
|
|
"epoch": 1.5882352941176472,
|
|
"frac_reward_zero_std": 0.5078125,
|
|
"grad_norm": 0.044641183127054146,
|
|
"learning_rate": 6.135314123594495e-07,
|
|
"loss": -0.0359,
|
|
"num_tokens": 155724383.0,
|
|
"reward": 0.5911458730697632,
|
|
"reward_std": 0.4917823374271393,
|
|
"rewards/reward_correctness/mean": 0.5911458134651184,
|
|
"rewards/reward_correctness/std": 0.49178236722946167,
|
|
"sampling/importance_sampling_ratio/max": 2.9951727390289307,
|
|
"sampling/importance_sampling_ratio/mean": 0.47802260518074036,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 13.149711608886719,
|
|
"sampling/sampling_logp_difference/mean": 0.03958186134696007,
|
|
"step": 108,
|
|
"step_time": 341.46006203070283
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1197916716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 3058.0,
|
|
"completions/mean_length": 961.5436401367188,
|
|
"completions/mean_terminated_length": 674.3380126953125,
|
|
"completions/min_length": 126.0,
|
|
"completions/min_terminated_length": 126.0,
|
|
"entropy": 0.44594370829872787,
|
|
"epoch": 1.6029411764705883,
|
|
"frac_reward_zero_std": 0.484375,
|
|
"grad_norm": 0.044574284241470793,
|
|
"learning_rate": 5.93087219203313e-07,
|
|
"loss": -0.0142,
|
|
"num_tokens": 157361190.0,
|
|
"reward": 0.5852864980697632,
|
|
"reward_std": 0.4928329288959503,
|
|
"rewards/reward_correctness/mean": 0.5852864384651184,
|
|
"rewards/reward_correctness/std": 0.4928329885005951,
|
|
"sampling/importance_sampling_ratio/max": 2.992027521133423,
|
|
"sampling/importance_sampling_ratio/mean": 0.5615074634552002,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.2640998363494873,
|
|
"sampling/sampling_logp_difference/mean": 0.025156300514936447,
|
|
"step": 109,
|
|
"step_time": 309.27131781354547
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.12109375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2381.0,
|
|
"completions/mean_length": 924.3776245117188,
|
|
"completions/mean_terminated_length": 628.4829711914062,
|
|
"completions/min_length": 141.0,
|
|
"completions/min_terminated_length": 141.0,
|
|
"entropy": 0.432338533049915,
|
|
"epoch": 1.6176470588235294,
|
|
"frac_reward_zero_std": 0.484375,
|
|
"grad_norm": 0.05142730527423491,
|
|
"learning_rate": 5.732508467514508e-07,
|
|
"loss": -0.024,
|
|
"num_tokens": 158932678.0,
|
|
"reward": 0.6119791865348816,
|
|
"reward_std": 0.4874580204486847,
|
|
"rewards/reward_correctness/mean": 0.6119791865348816,
|
|
"rewards/reward_correctness/std": 0.48745808005332947,
|
|
"sampling/importance_sampling_ratio/max": 2.987983465194702,
|
|
"sampling/importance_sampling_ratio/mean": 0.6046624779701233,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 3.0439860820770264,
|
|
"sampling/sampling_logp_difference/mean": 0.0254482664167881,
|
|
"step": 110,
|
|
"step_time": 326.00678399251774
|
|
},
|
|
{
|
|
"epoch": 1.6176470588235294,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.023809524519102915,
|
|
"eval_completions/max_length": 1239.75,
|
|
"eval_completions/max_terminated_length": 1011.1071428571429,
|
|
"eval_completions/mean_length": 626.029779343378,
|
|
"eval_completions/mean_terminated_length": 567.4938645135788,
|
|
"eval_completions/min_length": 269.6309523809524,
|
|
"eval_completions/min_terminated_length": 269.6309523809524,
|
|
"eval_entropy": 0.14750546021830468,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 158932678.0,
|
|
"eval_reward": 0.7182539875308672,
|
|
"eval_reward_std": 0.4125434688868977,
|
|
"eval_rewards/reward_correctness/mean": 0.7182539676626524,
|
|
"eval_rewards/reward_correctness/std": 0.41254346995126634,
|
|
"eval_runtime": 767.1044,
|
|
"eval_samples_per_second": 0.652,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.3713289199783633,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.08027123272439071,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.00021769897526041198,
|
|
"eval_sampling/sampling_logp_difference/max": 2.3605878239586238,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.037226093312104545,
|
|
"eval_steps_per_second": 0.11,
|
|
"step": 110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0904947966337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 1792.0,
|
|
"completions/mean_length": 836.53515625,
|
|
"completions/mean_terminated_length": 614.1087646484375,
|
|
"completions/min_length": 169.0,
|
|
"completions/min_terminated_length": 169.0,
|
|
"entropy": 0.36708237300626934,
|
|
"epoch": 1.6323529411764706,
|
|
"frac_reward_zero_std": 0.53125,
|
|
"grad_norm": 0.04335790136718892,
|
|
"learning_rate": 5.540337027170566e-07,
|
|
"loss": -0.0211,
|
|
"num_tokens": 160358860.0,
|
|
"reward": 0.6647135615348816,
|
|
"reward_std": 0.4722442328929901,
|
|
"rewards/reward_correctness/mean": 0.6647135615348816,
|
|
"rewards/reward_correctness/std": 0.4722442626953125,
|
|
"sampling/importance_sampling_ratio/max": 2.9934427738189697,
|
|
"sampling/importance_sampling_ratio/mean": 0.5987992882728577,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.4412472248077393,
|
|
"sampling/sampling_logp_difference/mean": 0.022612713277339935,
|
|
"step": 111,
|
|
"step_time": 305.83953939471394
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1041666716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2685.0,
|
|
"completions/mean_length": 878.3255615234375,
|
|
"completions/mean_terminated_length": 623.2470703125,
|
|
"completions/min_length": 148.0,
|
|
"completions/min_terminated_length": 148.0,
|
|
"entropy": 0.4083844933193177,
|
|
"epoch": 1.6470588235294117,
|
|
"frac_reward_zero_std": 0.5234375,
|
|
"grad_norm": 0.04585958475342479,
|
|
"learning_rate": 5.354468387008236e-07,
|
|
"loss": -0.0358,
|
|
"num_tokens": 161854512.0,
|
|
"reward": 0.6119791865348816,
|
|
"reward_std": 0.4874580502510071,
|
|
"rewards/reward_correctness/mean": 0.6119791865348816,
|
|
"rewards/reward_correctness/std": 0.48745808005332947,
|
|
"sampling/importance_sampling_ratio/max": 2.983163833618164,
|
|
"sampling/importance_sampling_ratio/mean": 0.5624591112136841,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.0098832845687866,
|
|
"sampling/sampling_logp_difference/mean": 0.024161145091056824,
|
|
"step": 112,
|
|
"step_time": 307.8114897273481
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0768229216337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2183.0,
|
|
"completions/mean_length": 788.8092651367188,
|
|
"completions/mean_terminated_length": 598.8201904296875,
|
|
"completions/min_length": 154.0,
|
|
"completions/min_terminated_length": 154.0,
|
|
"entropy": 0.34125655790558085,
|
|
"epoch": 1.6617647058823528,
|
|
"frac_reward_zero_std": 0.546875,
|
|
"grad_norm": 0.053624327974202725,
|
|
"learning_rate": 5.175009438352725e-07,
|
|
"loss": -0.0253,
|
|
"num_tokens": 163208983.0,
|
|
"reward": 0.65234375,
|
|
"reward_std": 0.4763812720775604,
|
|
"rewards/reward_correctness/mean": 0.65234375,
|
|
"rewards/reward_correctness/std": 0.4763812720775604,
|
|
"sampling/importance_sampling_ratio/max": 2.9941608905792236,
|
|
"sampling/importance_sampling_ratio/mean": 0.606112539768219,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.14524507522583,
|
|
"sampling/sampling_logp_difference/mean": 0.021032704040408134,
|
|
"step": 113,
|
|
"step_time": 300.7747645783238
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2575.0,
|
|
"completions/mean_length": 895.5358276367188,
|
|
"completions/mean_terminated_length": 670.3843383789062,
|
|
"completions/min_length": 150.0,
|
|
"completions/min_terminated_length": 150.0,
|
|
"entropy": 0.3677748629706912,
|
|
"epoch": 1.6764705882352942,
|
|
"frac_reward_zero_std": 0.4296875,
|
|
"grad_norm": 0.049732445330309516,
|
|
"learning_rate": 5.002063386375302e-07,
|
|
"loss": -0.0191,
|
|
"num_tokens": 164723246.0,
|
|
"reward": 0.6321614980697632,
|
|
"reward_std": 0.4823741316795349,
|
|
"rewards/reward_correctness/mean": 0.6321614384651184,
|
|
"rewards/reward_correctness/std": 0.4823741614818573,
|
|
"sampling/importance_sampling_ratio/max": 2.99418044090271,
|
|
"sampling/importance_sampling_ratio/mean": 0.5702483057975769,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 0.9949932098388672,
|
|
"sampling/sampling_logp_difference/mean": 0.021676061674952507,
|
|
"step": 114,
|
|
"step_time": 312.8656594133936
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0930989608168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2274.0,
|
|
"completions/mean_length": 818.5814208984375,
|
|
"completions/mean_terminated_length": 587.254150390625,
|
|
"completions/min_length": 161.0,
|
|
"completions/min_terminated_length": 161.0,
|
|
"entropy": 0.35392041283193976,
|
|
"epoch": 1.6911764705882353,
|
|
"frac_reward_zero_std": 0.5625,
|
|
"grad_norm": 0.0583397216318952,
|
|
"learning_rate": 4.835729690740971e-07,
|
|
"loss": -0.0117,
|
|
"num_tokens": 166113511.0,
|
|
"reward": 0.6751302480697632,
|
|
"reward_std": 0.4684787094593048,
|
|
"rewards/reward_correctness/mean": 0.6751301884651184,
|
|
"rewards/reward_correctness/std": 0.4684787094593048,
|
|
"sampling/importance_sampling_ratio/max": 2.9827520847320557,
|
|
"sampling/importance_sampling_ratio/mean": 0.6631502509117126,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 5.4962921142578125,
|
|
"sampling/sampling_logp_difference/mean": 0.02168840914964676,
|
|
"step": 115,
|
|
"step_time": 306.4326215046458
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1145833358168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2316.0,
|
|
"completions/mean_length": 907.9440307617188,
|
|
"completions/mean_terminated_length": 627.8897094726562,
|
|
"completions/min_length": 173.0,
|
|
"completions/min_terminated_length": 173.0,
|
|
"entropy": 0.44132739264750853,
|
|
"epoch": 1.7058823529411766,
|
|
"frac_reward_zero_std": 0.5078125,
|
|
"grad_norm": 0.05383767546988474,
|
|
"learning_rate": 4.6761040084101373e-07,
|
|
"loss": -0.0112,
|
|
"num_tokens": 167656493.0,
|
|
"reward": 0.5774739980697632,
|
|
"reward_std": 0.4941222071647644,
|
|
"rewards/reward_correctness/mean": 0.5774739384651184,
|
|
"rewards/reward_correctness/std": 0.494122177362442,
|
|
"sampling/importance_sampling_ratio/max": 2.9951207637786865,
|
|
"sampling/importance_sampling_ratio/mean": 0.5863032341003418,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 2.678741693496704,
|
|
"sampling/sampling_logp_difference/mean": 0.025177408009767532,
|
|
"step": 116,
|
|
"step_time": 310.71802368201315
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1243489608168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2546.0,
|
|
"completions/mean_length": 927.2233276367188,
|
|
"completions/mean_terminated_length": 622.8141479492188,
|
|
"completions/min_length": 163.0,
|
|
"completions/min_terminated_length": 163.0,
|
|
"entropy": 0.4399852986098267,
|
|
"epoch": 1.7205882352941178,
|
|
"frac_reward_zero_std": 0.5234375,
|
|
"grad_norm": 0.05001580981100338,
|
|
"learning_rate": 4.523278138627179e-07,
|
|
"loss": -0.0322,
|
|
"num_tokens": 169211412.0,
|
|
"reward": 0.591796875,
|
|
"reward_std": 0.4916611611843109,
|
|
"rewards/reward_correctness/mean": 0.591796875,
|
|
"rewards/reward_correctness/std": 0.4916611909866333,
|
|
"sampling/importance_sampling_ratio/max": 2.994056224822998,
|
|
"sampling/importance_sampling_ratio/mean": 0.5662473440170288,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.4523508548736572,
|
|
"sampling/sampling_logp_difference/mean": 0.026477819308638573,
|
|
"step": 117,
|
|
"step_time": 305.99031915096566
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1295572966337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2326.0,
|
|
"completions/mean_length": 996.1654052734375,
|
|
"completions/mean_terminated_length": 689.387451171875,
|
|
"completions/min_length": 124.0,
|
|
"completions/min_terminated_length": 124.0,
|
|
"entropy": 0.4670035591116175,
|
|
"epoch": 1.7352941176470589,
|
|
"frac_reward_zero_std": 0.4609375,
|
|
"grad_norm": 0.05290927155308445,
|
|
"learning_rate": 4.377339970127567e-07,
|
|
"loss": -0.0445,
|
|
"num_tokens": 170908298.0,
|
|
"reward": 0.5748698115348816,
|
|
"reward_std": 0.4945237338542938,
|
|
"rewards/reward_correctness/mean": 0.5748698115348816,
|
|
"rewards/reward_correctness/std": 0.4945237338542938,
|
|
"sampling/importance_sampling_ratio/max": 2.981295108795166,
|
|
"sampling/importance_sampling_ratio/mean": 0.4558072090148926,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 27.493650436401367,
|
|
"sampling/sampling_logp_difference/mean": 0.04102567210793495,
|
|
"step": 118,
|
|
"step_time": 352.26924694515765
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1126302108168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2728.0,
|
|
"completions/mean_length": 912.095703125,
|
|
"completions/mean_terminated_length": 637.9479370117188,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 0.38580389926210046,
|
|
"epoch": 1.75,
|
|
"frac_reward_zero_std": 0.4921875,
|
|
"grad_norm": 0.049328964216854024,
|
|
"learning_rate": 4.238373430593857e-07,
|
|
"loss": -0.0303,
|
|
"num_tokens": 172437473.0,
|
|
"reward": 0.646484375,
|
|
"reward_std": 0.47821667790412903,
|
|
"rewards/reward_correctness/mean": 0.646484375,
|
|
"rewards/reward_correctness/std": 0.4782167077064514,
|
|
"sampling/importance_sampling_ratio/max": 2.9586801528930664,
|
|
"sampling/importance_sampling_ratio/mean": 0.5879413485527039,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.0320615768432617,
|
|
"sampling/sampling_logp_difference/mean": 0.02208506129682064,
|
|
"step": 119,
|
|
"step_time": 327.5872627142817
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1178385466337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2830.0,
|
|
"completions/mean_length": 930.1575927734375,
|
|
"completions/mean_terminated_length": 644.065673828125,
|
|
"completions/min_length": 156.0,
|
|
"completions/min_terminated_length": 156.0,
|
|
"entropy": 0.41531214263523,
|
|
"epoch": 1.7647058823529411,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.055668120244077884,
|
|
"learning_rate": 4.1064584383896707e-07,
|
|
"loss": -0.0215,
|
|
"num_tokens": 174037783.0,
|
|
"reward": 0.6236979365348816,
|
|
"reward_std": 0.4846149981021881,
|
|
"rewards/reward_correctness/mean": 0.6236979365348816,
|
|
"rewards/reward_correctness/std": 0.4846150279045105,
|
|
"sampling/importance_sampling_ratio/max": 2.9956037998199463,
|
|
"sampling/importance_sampling_ratio/mean": 0.5683541893959045,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 6.9161248207092285,
|
|
"sampling/sampling_logp_difference/mean": 0.024076789617538452,
|
|
"step": 120,
|
|
"step_time": 326.06551372399554
|
|
},
|
|
{
|
|
"epoch": 1.7647058823529411,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.027777778605620067,
|
|
"eval_completions/max_length": 1323.1785714285713,
|
|
"eval_completions/max_terminated_length": 1018.6309523809524,
|
|
"eval_completions/mean_length": 642.468273344494,
|
|
"eval_completions/mean_terminated_length": 575.1000159127371,
|
|
"eval_completions/min_length": 269.6904761904762,
|
|
"eval_completions/min_terminated_length": 269.6904761904762,
|
|
"eval_entropy": 0.14365598683555922,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 174037783.0,
|
|
"eval_reward": 0.7242063674188796,
|
|
"eval_reward_std": 0.41008960952361423,
|
|
"eval_rewards/reward_correctness/mean": 0.7242063489698228,
|
|
"eval_rewards/reward_correctness/std": 0.41008960774966646,
|
|
"eval_runtime": 822.9839,
|
|
"eval_samples_per_second": 0.608,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.4101124081748172,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.086530597872687,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.00024189939948790175,
|
|
"eval_sampling/sampling_logp_difference/max": 2.3760013296490623,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.03615121694193,
|
|
"eval_steps_per_second": 0.102,
|
|
"step": 120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10546875,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2786.0,
|
|
"completions/mean_length": 862.91796875,
|
|
"completions/mean_terminated_length": 602.45849609375,
|
|
"completions/min_length": 107.0,
|
|
"completions/min_terminated_length": 107.0,
|
|
"entropy": 0.39749378041597083,
|
|
"epoch": 1.7794117647058822,
|
|
"frac_reward_zero_std": 0.546875,
|
|
"grad_norm": 0.043414378713222734,
|
|
"learning_rate": 3.9816708565993797e-07,
|
|
"loss": -0.0101,
|
|
"num_tokens": 175493701.0,
|
|
"reward": 0.6829427480697632,
|
|
"reward_std": 0.4654814600944519,
|
|
"rewards/reward_correctness/mean": 0.6829426884651184,
|
|
"rewards/reward_correctness/std": 0.4654814898967743,
|
|
"sampling/importance_sampling_ratio/max": 2.988877534866333,
|
|
"sampling/importance_sampling_ratio/mean": 0.5633504390716553,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.9039257764816284,
|
|
"sampling/sampling_logp_difference/mean": 0.023906981572508812,
|
|
"step": 121,
|
|
"step_time": 313.3631044062786
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0983072966337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2892.0,
|
|
"completions/mean_length": 852.6361083984375,
|
|
"completions/mean_terminated_length": 610.8281860351562,
|
|
"completions/min_length": 158.0,
|
|
"completions/min_terminated_length": 158.0,
|
|
"entropy": 0.35477442893898115,
|
|
"epoch": 1.7941176470588234,
|
|
"frac_reward_zero_std": 0.5078125,
|
|
"grad_norm": 0.067411019205484,
|
|
"learning_rate": 3.864082449399963e-07,
|
|
"loss": -0.014,
|
|
"num_tokens": 176947506.0,
|
|
"reward": 0.65234375,
|
|
"reward_std": 0.4763812720775604,
|
|
"rewards/reward_correctness/mean": 0.65234375,
|
|
"rewards/reward_correctness/std": 0.4763812720775604,
|
|
"sampling/importance_sampling_ratio/max": 2.9987151622772217,
|
|
"sampling/importance_sampling_ratio/mean": 0.6137019991874695,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.482698917388916,
|
|
"sampling/sampling_logp_difference/mean": 0.0226468276232481,
|
|
"step": 122,
|
|
"step_time": 307.41807026276365
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1080729216337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2622.0,
|
|
"completions/mean_length": 834.6829833984375,
|
|
"completions/mean_terminated_length": 563.7233276367188,
|
|
"completions/min_length": 126.0,
|
|
"completions/min_terminated_length": 126.0,
|
|
"entropy": 0.40907679829979315,
|
|
"epoch": 1.8088235294117647,
|
|
"frac_reward_zero_std": 0.453125,
|
|
"grad_norm": 0.0529649070159592,
|
|
"learning_rate": 3.753760840790081e-07,
|
|
"loss": -0.0265,
|
|
"num_tokens": 178387007.0,
|
|
"reward": 0.625,
|
|
"reward_std": 0.4842805564403534,
|
|
"rewards/reward_correctness/mean": 0.625,
|
|
"rewards/reward_correctness/std": 0.4842805862426758,
|
|
"sampling/importance_sampling_ratio/max": 2.9606382846832275,
|
|
"sampling/importance_sampling_ratio/mean": 0.5968267917633057,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.3054018020629883,
|
|
"sampling/sampling_logp_difference/mean": 0.025114521384239197,
|
|
"step": 123,
|
|
"step_time": 304.87532119173557
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1028645858168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2743.0,
|
|
"completions/mean_length": 843.2526245117188,
|
|
"completions/mean_terminated_length": 587.8563232421875,
|
|
"completions/min_length": 147.0,
|
|
"completions/min_terminated_length": 147.0,
|
|
"entropy": 0.3993370668613352,
|
|
"epoch": 1.8235294117647058,
|
|
"frac_reward_zero_std": 0.5859375,
|
|
"grad_norm": 0.05567172477716058,
|
|
"learning_rate": 3.650769475700163e-07,
|
|
"loss": -0.0468,
|
|
"num_tokens": 179818959.0,
|
|
"reward": 0.67578125,
|
|
"reward_std": 0.4682346284389496,
|
|
"rewards/reward_correctness/mean": 0.67578125,
|
|
"rewards/reward_correctness/std": 0.468234658241272,
|
|
"sampling/importance_sampling_ratio/max": 2.9983177185058594,
|
|
"sampling/importance_sampling_ratio/mean": 0.6072058081626892,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 2.201457977294922,
|
|
"sampling/sampling_logp_difference/mean": 0.02423350140452385,
|
|
"step": 124,
|
|
"step_time": 308.9692654227838
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1171875,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2944.0,
|
|
"completions/mean_length": 918.4173583984375,
|
|
"completions/mean_terminated_length": 632.5435180664062,
|
|
"completions/min_length": 141.0,
|
|
"completions/min_terminated_length": 141.0,
|
|
"entropy": 0.41694220766657963,
|
|
"epoch": 1.8382352941176472,
|
|
"frac_reward_zero_std": 0.46875,
|
|
"grad_norm": 0.04641479563046326,
|
|
"learning_rate": 3.5551675835057994e-07,
|
|
"loss": -0.0201,
|
|
"num_tokens": 181378256.0,
|
|
"reward": 0.6080729365348816,
|
|
"reward_std": 0.48833951354026794,
|
|
"rewards/reward_correctness/mean": 0.6080729365348816,
|
|
"rewards/reward_correctness/std": 0.48833954334259033,
|
|
"sampling/importance_sampling_ratio/max": 2.930337429046631,
|
|
"sampling/importance_sampling_ratio/mean": 0.5756347179412842,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 2.0342092514038086,
|
|
"sampling/sampling_logp_difference/mean": 0.024576259776949883,
|
|
"step": 125,
|
|
"step_time": 316.0315662450157
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.095703125,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2250.0,
|
|
"completions/mean_length": 848.2474365234375,
|
|
"completions/mean_terminated_length": 612.9042358398438,
|
|
"completions/min_length": 179.0,
|
|
"completions/min_terminated_length": 179.0,
|
|
"entropy": 0.3706806362606585,
|
|
"epoch": 1.8529411764705883,
|
|
"frac_reward_zero_std": 0.5546875,
|
|
"grad_norm": 0.04851792726196639,
|
|
"learning_rate": 3.4670101439654904e-07,
|
|
"loss": -0.0423,
|
|
"num_tokens": 182828080.0,
|
|
"reward": 0.5989583730697632,
|
|
"reward_std": 0.4902690052986145,
|
|
"rewards/reward_correctness/mean": 0.5989583134651184,
|
|
"rewards/reward_correctness/std": 0.4902690351009369,
|
|
"sampling/importance_sampling_ratio/max": 2.94360613822937,
|
|
"sampling/importance_sampling_ratio/mean": 0.5886579155921936,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.0113334655761719,
|
|
"sampling/sampling_logp_difference/mean": 0.02267294190824032,
|
|
"step": 126,
|
|
"step_time": 302.60534020280465
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1041666716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2757.0,
|
|
"completions/mean_length": 876.0866088867188,
|
|
"completions/mean_terminated_length": 620.747802734375,
|
|
"completions/min_length": 168.0,
|
|
"completions/min_terminated_length": 168.0,
|
|
"entropy": 0.3983296254882589,
|
|
"epoch": 1.8676470588235294,
|
|
"frac_reward_zero_std": 0.5234375,
|
|
"grad_norm": 0.06029741056238967,
|
|
"learning_rate": 3.3863478556022955e-07,
|
|
"loss": -0.0353,
|
|
"num_tokens": 184296533.0,
|
|
"reward": 0.642578125,
|
|
"reward_std": 0.47939661145210266,
|
|
"rewards/reward_correctness/mean": 0.642578125,
|
|
"rewards/reward_correctness/std": 0.47939661145210266,
|
|
"sampling/importance_sampling_ratio/max": 2.996951103210449,
|
|
"sampling/importance_sampling_ratio/mean": 0.5787960886955261,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.4283764362335205,
|
|
"sampling/sampling_logp_difference/mean": 0.023947691544890404,
|
|
"step": 127,
|
|
"step_time": 306.45986785087734
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1243489608168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2824.0,
|
|
"completions/mean_length": 918.0983276367188,
|
|
"completions/mean_terminated_length": 612.228271484375,
|
|
"completions/min_length": 180.0,
|
|
"completions/min_terminated_length": 180.0,
|
|
"entropy": 0.44061603699810803,
|
|
"epoch": 1.8823529411764706,
|
|
"frac_reward_zero_std": 0.453125,
|
|
"grad_norm": 0.042861531662967446,
|
|
"learning_rate": 3.313227106547582e-07,
|
|
"loss": -0.0345,
|
|
"num_tokens": 185860104.0,
|
|
"reward": 0.611328125,
|
|
"reward_std": 0.4876072406768799,
|
|
"rewards/reward_correctness/mean": 0.611328125,
|
|
"rewards/reward_correctness/std": 0.48760727047920227,
|
|
"sampling/importance_sampling_ratio/max": 2.9391822814941406,
|
|
"sampling/importance_sampling_ratio/mean": 0.5670685172080994,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.1543688774108887,
|
|
"sampling/sampling_logp_difference/mean": 0.025863494724035263,
|
|
"step": 128,
|
|
"step_time": 326.5879848697223
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1334635466337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 3031.0,
|
|
"completions/mean_length": 937.4264526367188,
|
|
"completions/mean_terminated_length": 610.2637329101562,
|
|
"completions/min_length": 157.0,
|
|
"completions/min_terminated_length": 157.0,
|
|
"entropy": 0.4768651365302503,
|
|
"epoch": 1.8970588235294117,
|
|
"frac_reward_zero_std": 0.5234375,
|
|
"grad_norm": 0.044436777773561156,
|
|
"learning_rate": 3.247689947863649e-07,
|
|
"loss": -0.029,
|
|
"num_tokens": 187473223.0,
|
|
"reward": 0.580078125,
|
|
"reward_std": 0.4937065541744232,
|
|
"rewards/reward_correctness/mean": 0.580078125,
|
|
"rewards/reward_correctness/std": 0.4937065839767456,
|
|
"sampling/importance_sampling_ratio/max": 2.9967188835144043,
|
|
"sampling/importance_sampling_ratio/mean": 0.6023716330528259,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.9644856452941895,
|
|
"sampling/sampling_logp_difference/mean": 0.027563076466321945,
|
|
"step": 129,
|
|
"step_time": 327.2371936151758
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1002604216337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2910.0,
|
|
"completions/mean_length": 883.4869995117188,
|
|
"completions/mean_terminated_length": 639.6150512695312,
|
|
"completions/min_length": 158.0,
|
|
"completions/min_terminated_length": 158.0,
|
|
"entropy": 0.3736990636680275,
|
|
"epoch": 1.9117647058823528,
|
|
"frac_reward_zero_std": 0.5234375,
|
|
"grad_norm": 0.04517983424365822,
|
|
"learning_rate": 3.1897740693605444e-07,
|
|
"loss": -0.0174,
|
|
"num_tokens": 188954447.0,
|
|
"reward": 0.6204427480697632,
|
|
"reward_std": 0.4854348301887512,
|
|
"rewards/reward_correctness/mean": 0.6204426884651184,
|
|
"rewards/reward_correctness/std": 0.4854348301887512,
|
|
"sampling/importance_sampling_ratio/max": 2.9764018058776855,
|
|
"sampling/importance_sampling_ratio/mean": 0.5933597683906555,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.9377039670944214,
|
|
"sampling/sampling_logp_difference/mean": 0.021819358691573143,
|
|
"step": 130,
|
|
"step_time": 302.9436299605295
|
|
},
|
|
{
|
|
"epoch": 1.9117647058823528,
|
|
"eval_clip_ratio/high_max": 0.0,
|
|
"eval_clip_ratio/high_mean": 0.0,
|
|
"eval_clip_ratio/low_mean": 0.0,
|
|
"eval_clip_ratio/low_min": 0.0,
|
|
"eval_clip_ratio/region_mean": 0.0,
|
|
"eval_completions/clipped_ratio": 0.02579365156236149,
|
|
"eval_completions/max_length": 1345.5,
|
|
"eval_completions/max_terminated_length": 1026.9285714285713,
|
|
"eval_completions/mean_length": 639.4682747977121,
|
|
"eval_completions/mean_terminated_length": 575.7262064615885,
|
|
"eval_completions/min_length": 265.5595238095238,
|
|
"eval_completions/min_terminated_length": 265.5595238095238,
|
|
"eval_entropy": 0.14163078057269254,
|
|
"eval_frac_reward_zero_std": 1.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 188954447.0,
|
|
"eval_reward": 0.7123016062236968,
|
|
"eval_reward_std": 0.4165747023764111,
|
|
"eval_rewards/reward_correctness/mean": 0.7123015891937983,
|
|
"eval_rewards/reward_correctness/std": 0.41657470166683197,
|
|
"eval_runtime": 839.8586,
|
|
"eval_samples_per_second": 0.595,
|
|
"eval_sampling/importance_sampling_ratio/max": 0.42427862378411063,
|
|
"eval_sampling/importance_sampling_ratio/mean": 0.08415341998373888,
|
|
"eval_sampling/importance_sampling_ratio/min": 0.00011745825359944689,
|
|
"eval_sampling/sampling_logp_difference/max": 2.4157612039929344,
|
|
"eval_sampling/sampling_logp_difference/mean": 0.0360281661685024,
|
|
"eval_steps_per_second": 0.1,
|
|
"step": 130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.09375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 3019.0,
|
|
"completions/mean_length": 856.2318115234375,
|
|
"completions/mean_terminated_length": 627.0143432617188,
|
|
"completions/min_length": 147.0,
|
|
"completions/min_terminated_length": 147.0,
|
|
"entropy": 0.35391040629474446,
|
|
"epoch": 1.9264705882352942,
|
|
"frac_reward_zero_std": 0.5390625,
|
|
"grad_norm": 0.05146181509009875,
|
|
"learning_rate": 3.1395127779210154e-07,
|
|
"loss": -0.002,
|
|
"num_tokens": 190413235.0,
|
|
"reward": 0.6484375,
|
|
"reward_std": 0.4776136577129364,
|
|
"rewards/reward_correctness/mean": 0.6484375,
|
|
"rewards/reward_correctness/std": 0.4776136875152588,
|
|
"sampling/importance_sampling_ratio/max": 2.9930741786956787,
|
|
"sampling/importance_sampling_ratio/mean": 0.6245337128639221,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.4557209014892578,
|
|
"sampling/sampling_logp_difference/mean": 0.021607233211398125,
|
|
"step": 131,
|
|
"step_time": 302.00397709198296
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1041666716337204,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2876.0,
|
|
"completions/mean_length": 940.916015625,
|
|
"completions/mean_terminated_length": 695.9607543945312,
|
|
"completions/min_length": 141.0,
|
|
"completions/min_terminated_length": 141.0,
|
|
"entropy": 0.38355993037112057,
|
|
"epoch": 1.9411764705882353,
|
|
"frac_reward_zero_std": 0.59375,
|
|
"grad_norm": 0.05004972084846401,
|
|
"learning_rate": 3.0969349783460157e-07,
|
|
"loss": -0.0302,
|
|
"num_tokens": 192014218.0,
|
|
"reward": 0.6432291865348816,
|
|
"reward_std": 0.47920235991477966,
|
|
"rewards/reward_correctness/mean": 0.6432291865348816,
|
|
"rewards/reward_correctness/std": 0.47920235991477966,
|
|
"sampling/importance_sampling_ratio/max": 2.9860193729400635,
|
|
"sampling/importance_sampling_ratio/mean": 0.5011322498321533,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 18.620969772338867,
|
|
"sampling/sampling_logp_difference/mean": 0.03865577280521393,
|
|
"step": 132,
|
|
"step_time": 352.00507430778816
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.08984375,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2799.0,
|
|
"completions/mean_length": 842.8984375,
|
|
"completions/mean_terminated_length": 622.9170532226562,
|
|
"completions/min_length": 133.0,
|
|
"completions/min_terminated_length": 133.0,
|
|
"entropy": 0.36497818934731185,
|
|
"epoch": 1.9558823529411766,
|
|
"frac_reward_zero_std": 0.515625,
|
|
"grad_norm": 0.07158180367926063,
|
|
"learning_rate": 3.0620651567318436e-07,
|
|
"loss": -0.0244,
|
|
"num_tokens": 193466782.0,
|
|
"reward": 0.6575521230697632,
|
|
"reward_std": 0.47468307614326477,
|
|
"rewards/reward_correctness/mean": 0.6575520634651184,
|
|
"rewards/reward_correctness/std": 0.47468310594558716,
|
|
"sampling/importance_sampling_ratio/max": 2.993088722229004,
|
|
"sampling/importance_sampling_ratio/mean": 0.5758584141731262,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.8104618787765503,
|
|
"sampling/sampling_logp_difference/mean": 0.022393621504306793,
|
|
"step": 133,
|
|
"step_time": 315.8437795029022
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.119140625,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2043.0,
|
|
"completions/mean_length": 923.3717651367188,
|
|
"completions/mean_terminated_length": 633.404296875,
|
|
"completions/min_length": 126.0,
|
|
"completions/min_terminated_length": 126.0,
|
|
"entropy": 0.42708491726079956,
|
|
"epoch": 1.9705882352941178,
|
|
"frac_reward_zero_std": 0.5234375,
|
|
"grad_norm": 0.06221349869734758,
|
|
"learning_rate": 3.0349233663883985e-07,
|
|
"loss": -0.0342,
|
|
"num_tokens": 195044597.0,
|
|
"reward": 0.6217448115348816,
|
|
"reward_std": 0.48510968685150146,
|
|
"rewards/reward_correctness/mean": 0.6217448115348816,
|
|
"rewards/reward_correctness/std": 0.48510968685150146,
|
|
"sampling/importance_sampling_ratio/max": 2.9823083877563477,
|
|
"sampling/importance_sampling_ratio/mean": 0.5720989108085632,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 3.544827461242676,
|
|
"sampling/sampling_logp_difference/mean": 0.02538115344941616,
|
|
"step": 134,
|
|
"step_time": 322.6157622030005
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1015625,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 2785.0,
|
|
"completions/mean_length": 868.0169677734375,
|
|
"completions/mean_terminated_length": 618.9080200195312,
|
|
"completions/min_length": 163.0,
|
|
"completions/min_terminated_length": 163.0,
|
|
"entropy": 0.4124655578052625,
|
|
"epoch": 1.9852941176470589,
|
|
"frac_reward_zero_std": 0.53125,
|
|
"grad_norm": 0.05529387617320365,
|
|
"learning_rate": 3.015525216306716e-07,
|
|
"loss": 0.0012,
|
|
"num_tokens": 196534291.0,
|
|
"reward": 0.6087239980697632,
|
|
"reward_std": 0.48819488286972046,
|
|
"rewards/reward_correctness/mean": 0.6087239384651184,
|
|
"rewards/reward_correctness/std": 0.48819488286972046,
|
|
"sampling/importance_sampling_ratio/max": 2.9879226684570312,
|
|
"sampling/importance_sampling_ratio/mean": 0.5620114803314209,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 1.6037323474884033,
|
|
"sampling/sampling_logp_difference/mean": 0.024546820670366287,
|
|
"step": 135,
|
|
"step_time": 314.64262982131913
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1067708358168602,
|
|
"completions/max_length": 3072.0,
|
|
"completions/max_terminated_length": 3046.0,
|
|
"completions/mean_length": 891.4622802734375,
|
|
"completions/mean_terminated_length": 630.8148803710938,
|
|
"completions/min_length": 175.0,
|
|
"completions/min_terminated_length": 175.0,
|
|
"entropy": 0.4176048967638053,
|
|
"epoch": 2.0,
|
|
"frac_reward_zero_std": 0.53125,
|
|
"grad_norm": 0.04742592778780676,
|
|
"learning_rate": 3.003881862182383e-07,
|
|
"loss": -0.0307,
|
|
"num_tokens": 198033861.0,
|
|
"reward": 0.6223958730697632,
|
|
"reward_std": 0.484945684671402,
|
|
"rewards/reward_correctness/mean": 0.6223958134651184,
|
|
"rewards/reward_correctness/std": 0.48494574427604675,
|
|
"sampling/importance_sampling_ratio/max": 2.9746880531311035,
|
|
"sampling/importance_sampling_ratio/mean": 0.5779362320899963,
|
|
"sampling/importance_sampling_ratio/min": 0.0,
|
|
"sampling/sampling_logp_difference/max": 4.9380927085876465,
|
|
"sampling/sampling_logp_difference/mean": 0.02337666228413582,
|
|
"step": 136,
|
|
"step_time": 300.921415225137
|
|
}
|
|
],
|
|
"logging_steps": 1.0,
|
|
"max_steps": 136,
|
|
"num_input_tokens_seen": 198033861,
|
|
"num_train_epochs": 2,
|
|
"save_steps": 10,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 2,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|