5255 lines
192 KiB
JSON
5255 lines
192 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.9,
|
|
"eval_steps": 500,
|
|
"global_step": 180,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 478.0,
|
|
"completions/max_terminated_length": 478.0,
|
|
"completions/mean_length": 191.0,
|
|
"completions/mean_terminated_length": 191.0,
|
|
"completions/min_length": 66.0,
|
|
"completions/min_terminated_length": 66.0,
|
|
"entropy": 0.5566893219947815,
|
|
"epoch": 0.005,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.370361328125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.24658073484897614,
|
|
"num_tokens": 10832.0,
|
|
"reward": -3.3000030517578125,
|
|
"reward_std": 85.62333679199219,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -3.4000015258789062,
|
|
"rewards/supergames_reward/std": 85.62333679199219,
|
|
"step": 1,
|
|
"step_time": 22.779711072013015
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 216.0,
|
|
"completions/max_terminated_length": 216.0,
|
|
"completions/mean_length": 131.5,
|
|
"completions/mean_terminated_length": 131.5,
|
|
"completions/min_length": 60.0,
|
|
"completions/min_terminated_length": 60.0,
|
|
"entropy": 0.6395841240882874,
|
|
"epoch": 0.01,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 8.09774398803711,
|
|
"learning_rate": 9.95e-07,
|
|
"loss": 0.20567649602890015,
|
|
"num_tokens": 16404.0,
|
|
"reward": -12.422499656677246,
|
|
"reward_std": 7.134707450866699,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -12.522500038146973,
|
|
"rewards/supergames_reward/std": 7.134707927703857,
|
|
"step": 2,
|
|
"step_time": 9.960156448010821
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.0,
|
|
"completions/max_terminated_length": 287.0,
|
|
"completions/mean_length": 156.625,
|
|
"completions/mean_terminated_length": 156.625,
|
|
"completions/min_length": 77.0,
|
|
"completions/min_terminated_length": 77.0,
|
|
"entropy": 0.562222421169281,
|
|
"epoch": 0.015,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.9053616523742676,
|
|
"learning_rate": 9.9e-07,
|
|
"loss": 0.09602774679660797,
|
|
"num_tokens": 26953.0,
|
|
"reward": -74.94999694824219,
|
|
"reward_std": 70.73094177246094,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -75.0,
|
|
"rewards/supergames_reward/std": 70.71067810058594,
|
|
"step": 3,
|
|
"step_time": 13.686320498993155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 268.0,
|
|
"completions/max_terminated_length": 268.0,
|
|
"completions/mean_length": 154.125,
|
|
"completions/mean_terminated_length": 154.125,
|
|
"completions/min_length": 53.0,
|
|
"completions/min_terminated_length": 53.0,
|
|
"entropy": 0.7323317527770996,
|
|
"epoch": 0.02,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.933310031890869,
|
|
"learning_rate": 9.849999999999999e-07,
|
|
"loss": 0.28249427676200867,
|
|
"num_tokens": 36514.0,
|
|
"reward": -40.7599983215332,
|
|
"reward_std": 81.73140716552734,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -40.80999755859375,
|
|
"rewards/supergames_reward/std": 81.68995666503906,
|
|
"step": 4,
|
|
"step_time": 12.805880262021674
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 401.0,
|
|
"completions/max_terminated_length": 401.0,
|
|
"completions/mean_length": 205.25,
|
|
"completions/mean_terminated_length": 205.25,
|
|
"completions/min_length": 115.0,
|
|
"completions/min_terminated_length": 115.0,
|
|
"entropy": 0.6167430281639099,
|
|
"epoch": 0.025,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.344135046005249,
|
|
"learning_rate": 9.8e-07,
|
|
"loss": 0.1433994024991989,
|
|
"num_tokens": 47476.0,
|
|
"reward": -97.6612548828125,
|
|
"reward_std": 6.3321428298950195,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -97.76124572753906,
|
|
"rewards/supergames_reward/std": 6.3321428298950195,
|
|
"step": 5,
|
|
"step_time": 18.789364666008623
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.0,
|
|
"completions/max_terminated_length": 276.0,
|
|
"completions/mean_length": 167.25,
|
|
"completions/mean_terminated_length": 167.25,
|
|
"completions/min_length": 73.0,
|
|
"completions/min_terminated_length": 73.0,
|
|
"entropy": 0.7583790421485901,
|
|
"epoch": 0.03,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.172371864318848,
|
|
"learning_rate": 9.75e-07,
|
|
"loss": -0.12127404659986496,
|
|
"num_tokens": 55246.0,
|
|
"reward": -58.687503814697266,
|
|
"reward_std": 58.64059829711914,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -58.76250457763672,
|
|
"rewards/supergames_reward/std": 58.620460510253906,
|
|
"step": 6,
|
|
"step_time": 12.948570273991209
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 261.0,
|
|
"completions/max_terminated_length": 261.0,
|
|
"completions/mean_length": 133.875,
|
|
"completions/mean_terminated_length": 133.875,
|
|
"completions/min_length": 57.0,
|
|
"completions/min_terminated_length": 57.0,
|
|
"entropy": 0.7447654604911804,
|
|
"epoch": 0.035,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.428595066070557,
|
|
"learning_rate": 9.7e-07,
|
|
"loss": 0.10135584324598312,
|
|
"num_tokens": 60837.0,
|
|
"reward": -32.45000076293945,
|
|
"reward_std": 41.754486083984375,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -32.5,
|
|
"rewards/supergames_reward/std": 41.66190719604492,
|
|
"step": 7,
|
|
"step_time": 12.058315072004916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 414.0,
|
|
"completions/max_terminated_length": 414.0,
|
|
"completions/mean_length": 209.125,
|
|
"completions/mean_terminated_length": 209.125,
|
|
"completions/min_length": 104.0,
|
|
"completions/min_terminated_length": 104.0,
|
|
"entropy": 0.3860666751861572,
|
|
"epoch": 0.04,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.806255578994751,
|
|
"learning_rate": 9.649999999999999e-07,
|
|
"loss": -0.33238139748573303,
|
|
"num_tokens": 70838.0,
|
|
"reward": -80.50375366210938,
|
|
"reward_std": 38.82380294799805,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -80.60375213623047,
|
|
"rewards/supergames_reward/std": 38.82379913330078,
|
|
"step": 8,
|
|
"step_time": 19.323370319994865
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 199.0,
|
|
"completions/mean_length": 247.125,
|
|
"completions/mean_terminated_length": 158.83334350585938,
|
|
"completions/min_length": 106.0,
|
|
"completions/min_terminated_length": 106.0,
|
|
"entropy": 0.5816237330436707,
|
|
"epoch": 0.045,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.3209943771362305,
|
|
"learning_rate": 9.6e-07,
|
|
"loss": 0.36752766370773315,
|
|
"num_tokens": 82135.0,
|
|
"reward": -27.688751220703125,
|
|
"reward_std": 82.8358154296875,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -27.738750457763672,
|
|
"rewards/supergames_reward/std": 82.78590393066406,
|
|
"step": 9,
|
|
"step_time": 21.948575104994234
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 240.0,
|
|
"completions/max_terminated_length": 240.0,
|
|
"completions/mean_length": 153.0,
|
|
"completions/mean_terminated_length": 153.0,
|
|
"completions/min_length": 51.0,
|
|
"completions/min_terminated_length": 51.0,
|
|
"entropy": 0.6301521062850952,
|
|
"epoch": 0.05,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.125359058380127,
|
|
"learning_rate": 9.55e-07,
|
|
"loss": 0.06129350885748863,
|
|
"num_tokens": 89807.0,
|
|
"reward": -78.39624786376953,
|
|
"reward_std": 40.17042922973633,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -78.44625091552734,
|
|
"rewards/supergames_reward/std": 40.1396484375,
|
|
"step": 10,
|
|
"step_time": 9.098202993016457
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 133.0,
|
|
"completions/max_terminated_length": 133.0,
|
|
"completions/mean_length": 112.0,
|
|
"completions/mean_terminated_length": 112.0,
|
|
"completions/min_length": 68.0,
|
|
"completions/min_terminated_length": 68.0,
|
|
"entropy": 0.5521990656852722,
|
|
"epoch": 0.055,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.616060256958008,
|
|
"learning_rate": 9.499999999999999e-07,
|
|
"loss": -0.006313305348157883,
|
|
"num_tokens": 95231.0,
|
|
"reward": -12.161249160766602,
|
|
"reward_std": 6.39578104019165,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -12.261249542236328,
|
|
"rewards/supergames_reward/std": 6.39578104019165,
|
|
"step": 11,
|
|
"step_time": 5.136311663984088
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 396.0,
|
|
"completions/max_terminated_length": 396.0,
|
|
"completions/mean_length": 214.875,
|
|
"completions/mean_terminated_length": 214.875,
|
|
"completions/min_length": 55.0,
|
|
"completions/min_terminated_length": 55.0,
|
|
"entropy": 0.5505905151367188,
|
|
"epoch": 0.06,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.289583206176758,
|
|
"learning_rate": 9.45e-07,
|
|
"loss": -0.29802003502845764,
|
|
"num_tokens": 106238.0,
|
|
"reward": -91.66999816894531,
|
|
"reward_std": 23.2779541015625,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -91.77000427246094,
|
|
"rewards/supergames_reward/std": 23.277956008911133,
|
|
"step": 12,
|
|
"step_time": 14.945365622988902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.0,
|
|
"completions/max_terminated_length": 269.0,
|
|
"completions/mean_length": 167.375,
|
|
"completions/mean_terminated_length": 167.375,
|
|
"completions/min_length": 113.0,
|
|
"completions/min_terminated_length": 113.0,
|
|
"entropy": 0.540415346622467,
|
|
"epoch": 0.065,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.858291149139404,
|
|
"learning_rate": 9.399999999999999e-07,
|
|
"loss": 0.24004128575325012,
|
|
"num_tokens": 116937.0,
|
|
"reward": -20.38249969482422,
|
|
"reward_std": 87.55204010009766,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -20.457500457763672,
|
|
"rewards/supergames_reward/std": 87.52605438232422,
|
|
"step": 13,
|
|
"step_time": 10.482285185018554
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 254.0,
|
|
"completions/max_terminated_length": 254.0,
|
|
"completions/mean_length": 156.875,
|
|
"completions/mean_terminated_length": 156.875,
|
|
"completions/min_length": 114.0,
|
|
"completions/min_terminated_length": 114.0,
|
|
"entropy": 0.7237679958343506,
|
|
"epoch": 0.07,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.083980560302734,
|
|
"learning_rate": 9.35e-07,
|
|
"loss": -0.19025824964046478,
|
|
"num_tokens": 124640.0,
|
|
"reward": -51.397499084472656,
|
|
"reward_std": 54.486454010009766,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -51.49749755859375,
|
|
"rewards/supergames_reward/std": 54.486454010009766,
|
|
"step": 14,
|
|
"step_time": 9.597231683001155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 236.0,
|
|
"completions/max_terminated_length": 236.0,
|
|
"completions/mean_length": 141.375,
|
|
"completions/mean_terminated_length": 141.375,
|
|
"completions/min_length": 91.0,
|
|
"completions/min_terminated_length": 91.0,
|
|
"entropy": 0.754334568977356,
|
|
"epoch": 0.075,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.720428943634033,
|
|
"learning_rate": 9.3e-07,
|
|
"loss": -0.16830675303936005,
|
|
"num_tokens": 132219.0,
|
|
"reward": -56.226253509521484,
|
|
"reward_std": 60.115657806396484,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -56.32625198364258,
|
|
"rewards/supergames_reward/std": 60.11566162109375,
|
|
"step": 15,
|
|
"step_time": 8.998362872982398
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 209.0,
|
|
"completions/max_terminated_length": 209.0,
|
|
"completions/mean_length": 136.375,
|
|
"completions/mean_terminated_length": 136.375,
|
|
"completions/min_length": 97.0,
|
|
"completions/min_terminated_length": 97.0,
|
|
"entropy": 0.6280568838119507,
|
|
"epoch": 0.08,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.9681806564331055,
|
|
"learning_rate": 9.25e-07,
|
|
"loss": -0.01910916529595852,
|
|
"num_tokens": 139726.0,
|
|
"reward": -68.24000549316406,
|
|
"reward_std": 52.238914489746094,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -68.33999633789062,
|
|
"rewards/supergames_reward/std": 52.238914489746094,
|
|
"step": 16,
|
|
"step_time": 8.034480294008972
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 365.0,
|
|
"completions/max_terminated_length": 365.0,
|
|
"completions/mean_length": 211.875,
|
|
"completions/mean_terminated_length": 211.875,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"entropy": 0.899176836013794,
|
|
"epoch": 0.085,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.691746711730957,
|
|
"learning_rate": 9.2e-07,
|
|
"loss": 0.25513291358947754,
|
|
"num_tokens": 147869.0,
|
|
"reward": -99.92500305175781,
|
|
"reward_std": 0.07070883363485336,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -100.0,
|
|
"rewards/supergames_reward/std": 0.0,
|
|
"step": 17,
|
|
"step_time": 13.655792869016295
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 387.0,
|
|
"completions/max_terminated_length": 387.0,
|
|
"completions/mean_length": 238.0,
|
|
"completions/mean_terminated_length": 238.0,
|
|
"completions/min_length": 87.0,
|
|
"completions/min_terminated_length": 87.0,
|
|
"entropy": 0.5596873760223389,
|
|
"epoch": 0.09,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.6711347103118896,
|
|
"learning_rate": 9.15e-07,
|
|
"loss": -0.2628335654735565,
|
|
"num_tokens": 159101.0,
|
|
"reward": -24.900001525878906,
|
|
"reward_std": 103.50983428955078,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -25.0,
|
|
"rewards/supergames_reward/std": 103.50983428955078,
|
|
"step": 18,
|
|
"step_time": 14.784329420013819
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 277.0,
|
|
"completions/mean_length": 219.75,
|
|
"completions/mean_terminated_length": 178.00001525878906,
|
|
"completions/min_length": 90.0,
|
|
"completions/min_terminated_length": 90.0,
|
|
"entropy": 0.9245517253875732,
|
|
"epoch": 0.095,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.55502986907959,
|
|
"learning_rate": 9.1e-07,
|
|
"loss": 0.13060179352760315,
|
|
"num_tokens": 167299.0,
|
|
"reward": -86.89125061035156,
|
|
"reward_std": 36.95603561401367,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -86.94125366210938,
|
|
"rewards/supergames_reward/std": 36.93572235107422,
|
|
"step": 19,
|
|
"step_time": 18.832684700988466
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 267.0,
|
|
"completions/max_terminated_length": 267.0,
|
|
"completions/mean_length": 177.25,
|
|
"completions/mean_terminated_length": 177.25,
|
|
"completions/min_length": 93.0,
|
|
"completions/min_terminated_length": 93.0,
|
|
"entropy": 0.7331254482269287,
|
|
"epoch": 0.1,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.538497447967529,
|
|
"learning_rate": 9.05e-07,
|
|
"loss": -0.16418980062007904,
|
|
"num_tokens": 175141.0,
|
|
"reward": -68.52874755859375,
|
|
"reward_std": 58.48719787597656,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -68.62875366210938,
|
|
"rewards/supergames_reward/std": 58.48720169067383,
|
|
"step": 20,
|
|
"step_time": 10.090975169994636
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.0,
|
|
"completions/max_terminated_length": 300.0,
|
|
"completions/mean_length": 190.625,
|
|
"completions/mean_terminated_length": 190.625,
|
|
"completions/min_length": 103.0,
|
|
"completions/min_terminated_length": 103.0,
|
|
"entropy": 0.6659425497055054,
|
|
"epoch": 0.105,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.485373497009277,
|
|
"learning_rate": 9e-07,
|
|
"loss": -0.10106432437896729,
|
|
"num_tokens": 183098.0,
|
|
"reward": -71.48875427246094,
|
|
"reward_std": 55.33565139770508,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -71.5887451171875,
|
|
"rewards/supergames_reward/std": 55.33565139770508,
|
|
"step": 21,
|
|
"step_time": 11.255001295008697
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.0,
|
|
"completions/max_terminated_length": 279.0,
|
|
"completions/mean_length": 179.0,
|
|
"completions/mean_terminated_length": 179.0,
|
|
"completions/min_length": 103.0,
|
|
"completions/min_terminated_length": 103.0,
|
|
"entropy": 0.49365949630737305,
|
|
"epoch": 0.11,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.829419136047363,
|
|
"learning_rate": 8.95e-07,
|
|
"loss": 0.10587231814861298,
|
|
"num_tokens": 192826.0,
|
|
"reward": -75.62000274658203,
|
|
"reward_std": 45.01959991455078,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -75.69499969482422,
|
|
"rewards/supergames_reward/std": 45.00411605834961,
|
|
"step": 22,
|
|
"step_time": 10.611246599000879
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.0,
|
|
"completions/max_terminated_length": 315.0,
|
|
"completions/mean_length": 212.625,
|
|
"completions/mean_terminated_length": 212.625,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"entropy": 0.43159469962120056,
|
|
"epoch": 0.115,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.541125774383545,
|
|
"learning_rate": 8.9e-07,
|
|
"loss": -0.1681259125471115,
|
|
"num_tokens": 202815.0,
|
|
"reward": 23.67624855041504,
|
|
"reward_std": 67.74340057373047,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 23.576250076293945,
|
|
"rewards/supergames_reward/std": 67.743408203125,
|
|
"step": 23,
|
|
"step_time": 11.731771531980485
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 168.0,
|
|
"completions/max_terminated_length": 168.0,
|
|
"completions/mean_length": 101.625,
|
|
"completions/mean_terminated_length": 101.625,
|
|
"completions/min_length": 65.0,
|
|
"completions/min_terminated_length": 65.0,
|
|
"entropy": 0.3972298502922058,
|
|
"epoch": 0.12,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.8285417556762695,
|
|
"learning_rate": 8.85e-07,
|
|
"loss": -0.011585958302021027,
|
|
"num_tokens": 208188.0,
|
|
"reward": 58.849998474121094,
|
|
"reward_std": 56.9304084777832,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 58.75,
|
|
"rewards/supergames_reward/std": 56.9304084777832,
|
|
"step": 24,
|
|
"step_time": 6.187504950998118
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.0,
|
|
"completions/max_terminated_length": 303.0,
|
|
"completions/mean_length": 165.375,
|
|
"completions/mean_terminated_length": 165.375,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"entropy": 0.6339899301528931,
|
|
"epoch": 0.125,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.823799133300781,
|
|
"learning_rate": 8.799999999999999e-07,
|
|
"loss": 0.05159185826778412,
|
|
"num_tokens": 215935.0,
|
|
"reward": -23.521251678466797,
|
|
"reward_std": 74.23067474365234,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -23.621248245239258,
|
|
"rewards/supergames_reward/std": 74.23067474365234,
|
|
"step": 25,
|
|
"step_time": 11.059416300005978
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 349.0,
|
|
"completions/max_terminated_length": 349.0,
|
|
"completions/mean_length": 204.0,
|
|
"completions/mean_terminated_length": 204.0,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 0.6462895274162292,
|
|
"epoch": 0.13,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.969292640686035,
|
|
"learning_rate": 8.75e-07,
|
|
"loss": 0.0115677984431386,
|
|
"num_tokens": 223999.0,
|
|
"reward": -74.8175048828125,
|
|
"reward_std": 40.94221115112305,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -74.89250183105469,
|
|
"rewards/supergames_reward/std": 40.924625396728516,
|
|
"step": 26,
|
|
"step_time": 12.666237785975682
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.0,
|
|
"completions/max_terminated_length": 299.0,
|
|
"completions/mean_length": 166.5,
|
|
"completions/mean_terminated_length": 166.5,
|
|
"completions/min_length": 87.0,
|
|
"completions/min_terminated_length": 87.0,
|
|
"entropy": 0.4756850600242615,
|
|
"epoch": 0.135,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.915069103240967,
|
|
"learning_rate": 8.699999999999999e-07,
|
|
"loss": -0.13321346044540405,
|
|
"num_tokens": 233643.0,
|
|
"reward": -52.125,
|
|
"reward_std": 19.405282974243164,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -52.20000076293945,
|
|
"rewards/supergames_reward/std": 19.33464813232422,
|
|
"step": 27,
|
|
"step_time": 11.172539752995363
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 184.0,
|
|
"completions/max_terminated_length": 184.0,
|
|
"completions/mean_length": 137.125,
|
|
"completions/mean_terminated_length": 137.125,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"entropy": 0.5834778547286987,
|
|
"epoch": 0.14,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.146666526794434,
|
|
"learning_rate": 8.65e-07,
|
|
"loss": 0.060454584658145905,
|
|
"num_tokens": 241204.0,
|
|
"reward": -68.99250793457031,
|
|
"reward_std": 51.010581970214844,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -69.09249877929688,
|
|
"rewards/supergames_reward/std": 51.010581970214844,
|
|
"step": 28,
|
|
"step_time": 7.025046669004951
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 331.0,
|
|
"completions/max_terminated_length": 331.0,
|
|
"completions/mean_length": 182.375,
|
|
"completions/mean_terminated_length": 182.375,
|
|
"completions/min_length": 106.0,
|
|
"completions/min_terminated_length": 106.0,
|
|
"entropy": 0.6368551254272461,
|
|
"epoch": 0.145,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.15535306930542,
|
|
"learning_rate": 8.599999999999999e-07,
|
|
"loss": -0.3323673903942108,
|
|
"num_tokens": 249087.0,
|
|
"reward": -44.17000198364258,
|
|
"reward_std": 80.5373306274414,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -44.27000045776367,
|
|
"rewards/supergames_reward/std": 80.5373306274414,
|
|
"step": 29,
|
|
"step_time": 12.079259724996518
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 190.0,
|
|
"completions/max_terminated_length": 190.0,
|
|
"completions/mean_length": 127.75,
|
|
"completions/mean_terminated_length": 127.75,
|
|
"completions/min_length": 80.0,
|
|
"completions/min_terminated_length": 80.0,
|
|
"entropy": 0.4491196870803833,
|
|
"epoch": 0.15,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.896186828613281,
|
|
"learning_rate": 8.55e-07,
|
|
"loss": -0.12887656688690186,
|
|
"num_tokens": 259445.0,
|
|
"reward": 57.446250915527344,
|
|
"reward_std": 38.78837585449219,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 57.34625244140625,
|
|
"rewards/supergames_reward/std": 38.78837966918945,
|
|
"step": 30,
|
|
"step_time": 7.483972909016302
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 166.0,
|
|
"completions/max_terminated_length": 166.0,
|
|
"completions/mean_length": 112.875,
|
|
"completions/mean_terminated_length": 112.875,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 0.4858454465866089,
|
|
"epoch": 0.155,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.742906093597412,
|
|
"learning_rate": 8.499999999999999e-07,
|
|
"loss": 0.06930096447467804,
|
|
"num_tokens": 264924.0,
|
|
"reward": -21.174999237060547,
|
|
"reward_std": 31.89051628112793,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -21.25,
|
|
"rewards/supergames_reward/std": 31.819807052612305,
|
|
"step": 31,
|
|
"step_time": 6.1866529019898735
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 148.0,
|
|
"completions/max_terminated_length": 148.0,
|
|
"completions/mean_length": 118.125,
|
|
"completions/mean_terminated_length": 118.125,
|
|
"completions/min_length": 98.0,
|
|
"completions/min_terminated_length": 98.0,
|
|
"entropy": 0.45428749918937683,
|
|
"epoch": 0.16,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.663414478302002,
|
|
"learning_rate": 8.45e-07,
|
|
"loss": 0.05735419690608978,
|
|
"num_tokens": 270381.0,
|
|
"reward": 0.4687504768371582,
|
|
"reward_std": 38.66371154785156,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 0.36875057220458984,
|
|
"rewards/supergames_reward/std": 38.6637077331543,
|
|
"step": 32,
|
|
"step_time": 5.577042757999152
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 345.0,
|
|
"completions/max_terminated_length": 345.0,
|
|
"completions/mean_length": 166.875,
|
|
"completions/mean_terminated_length": 166.875,
|
|
"completions/min_length": 86.0,
|
|
"completions/min_terminated_length": 86.0,
|
|
"entropy": 0.6107293367385864,
|
|
"epoch": 0.165,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.75006628036499,
|
|
"learning_rate": 8.399999999999999e-07,
|
|
"loss": -0.14760127663612366,
|
|
"num_tokens": 278156.0,
|
|
"reward": -87.84750366210938,
|
|
"reward_std": 26.84708023071289,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -87.94749450683594,
|
|
"rewards/supergames_reward/std": 26.847076416015625,
|
|
"step": 33,
|
|
"step_time": 12.552876825997373
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 360.0,
|
|
"completions/mean_length": 231.0,
|
|
"completions/mean_terminated_length": 190.85714721679688,
|
|
"completions/min_length": 74.0,
|
|
"completions/min_terminated_length": 74.0,
|
|
"entropy": 0.6089653372764587,
|
|
"epoch": 0.17,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.781267166137695,
|
|
"learning_rate": 8.349999999999999e-07,
|
|
"loss": 0.4294504225254059,
|
|
"num_tokens": 289364.0,
|
|
"reward": -99.92500305175781,
|
|
"reward_std": 0.07070883363485336,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -100.0,
|
|
"rewards/supergames_reward/std": 0.0,
|
|
"step": 34,
|
|
"step_time": 18.961819477990502
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 218.0,
|
|
"completions/max_terminated_length": 218.0,
|
|
"completions/mean_length": 134.125,
|
|
"completions/mean_terminated_length": 134.125,
|
|
"completions/min_length": 104.0,
|
|
"completions/min_terminated_length": 104.0,
|
|
"entropy": 0.46158918738365173,
|
|
"epoch": 0.175,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.016353607177734,
|
|
"learning_rate": 8.299999999999999e-07,
|
|
"loss": 0.09251818805932999,
|
|
"num_tokens": 298733.0,
|
|
"reward": -35.558753967285156,
|
|
"reward_std": 15.881204605102539,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -35.65875244140625,
|
|
"rewards/supergames_reward/std": 15.881203651428223,
|
|
"step": 35,
|
|
"step_time": 8.38491243700264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 207.0,
|
|
"completions/max_terminated_length": 207.0,
|
|
"completions/mean_length": 156.875,
|
|
"completions/mean_terminated_length": 156.875,
|
|
"completions/min_length": 111.0,
|
|
"completions/min_terminated_length": 111.0,
|
|
"entropy": 0.5104788541793823,
|
|
"epoch": 0.18,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.592893123626709,
|
|
"learning_rate": 8.249999999999999e-07,
|
|
"loss": -0.0410832017660141,
|
|
"num_tokens": 306420.0,
|
|
"reward": 5.4212493896484375,
|
|
"reward_std": 93.50404357910156,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 5.346250534057617,
|
|
"rewards/supergames_reward/std": 93.4718246459961,
|
|
"step": 36,
|
|
"step_time": 7.811868985998444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 241.0,
|
|
"completions/max_terminated_length": 241.0,
|
|
"completions/mean_length": 178.0,
|
|
"completions/mean_terminated_length": 178.0,
|
|
"completions/min_length": 106.0,
|
|
"completions/min_terminated_length": 106.0,
|
|
"entropy": 0.7527878880500793,
|
|
"epoch": 0.185,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.8812546730041504,
|
|
"learning_rate": 8.199999999999999e-07,
|
|
"loss": 0.12539231777191162,
|
|
"num_tokens": 314252.0,
|
|
"reward": -89.58500671386719,
|
|
"reward_std": 29.256120681762695,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -89.66000366210938,
|
|
"rewards/supergames_reward/std": 29.24593734741211,
|
|
"step": 37,
|
|
"step_time": 8.959742914012168
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 199.0,
|
|
"completions/max_terminated_length": 199.0,
|
|
"completions/mean_length": 133.375,
|
|
"completions/mean_terminated_length": 133.375,
|
|
"completions/min_length": 87.0,
|
|
"completions/min_terminated_length": 87.0,
|
|
"entropy": 0.3347901403903961,
|
|
"epoch": 0.19,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.205414295196533,
|
|
"learning_rate": 8.149999999999999e-07,
|
|
"loss": -0.06991486996412277,
|
|
"num_tokens": 319839.0,
|
|
"reward": -14.498749732971191,
|
|
"reward_std": 13.007229804992676,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -14.598750114440918,
|
|
"rewards/supergames_reward/std": 13.007229804992676,
|
|
"step": 38,
|
|
"step_time": 7.377183554985095
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 181.0,
|
|
"completions/max_terminated_length": 181.0,
|
|
"completions/mean_length": 114.375,
|
|
"completions/mean_terminated_length": 114.375,
|
|
"completions/min_length": 58.0,
|
|
"completions/min_terminated_length": 58.0,
|
|
"entropy": 0.4445965886116028,
|
|
"epoch": 0.195,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.561298847198486,
|
|
"learning_rate": 8.1e-07,
|
|
"loss": -0.06235164776444435,
|
|
"num_tokens": 325290.0,
|
|
"reward": -7.425000190734863,
|
|
"reward_std": 53.701602935791016,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -7.5,
|
|
"rewards/supergames_reward/std": 53.652320861816406,
|
|
"step": 39,
|
|
"step_time": 6.753862089011818
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 370.0,
|
|
"completions/max_terminated_length": 370.0,
|
|
"completions/mean_length": 187.625,
|
|
"completions/mean_terminated_length": 187.625,
|
|
"completions/min_length": 60.0,
|
|
"completions/min_terminated_length": 60.0,
|
|
"entropy": 0.3950234353542328,
|
|
"epoch": 0.2,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.5988712310791016,
|
|
"learning_rate": 8.05e-07,
|
|
"loss": -0.1381440907716751,
|
|
"num_tokens": 335119.0,
|
|
"reward": -33.68375015258789,
|
|
"reward_std": 42.069435119628906,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -33.783748626708984,
|
|
"rewards/supergames_reward/std": 42.069435119628906,
|
|
"step": 40,
|
|
"step_time": 13.718958162004128
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 371.0,
|
|
"completions/mean_length": 251.125,
|
|
"completions/mean_terminated_length": 213.85714721679688,
|
|
"completions/min_length": 121.0,
|
|
"completions/min_terminated_length": 121.0,
|
|
"entropy": 0.507067084312439,
|
|
"epoch": 0.205,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.3898658752441406,
|
|
"learning_rate": 8e-07,
|
|
"loss": 0.0078964838758111,
|
|
"num_tokens": 346464.0,
|
|
"reward": -86.48625183105469,
|
|
"reward_std": 38.020694732666016,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -86.56124877929688,
|
|
"rewards/supergames_reward/std": 38.010528564453125,
|
|
"step": 41,
|
|
"step_time": 18.90960379401804
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 152.0,
|
|
"completions/max_terminated_length": 152.0,
|
|
"completions/mean_length": 116.375,
|
|
"completions/mean_terminated_length": 116.375,
|
|
"completions/min_length": 93.0,
|
|
"completions/min_terminated_length": 93.0,
|
|
"entropy": 0.3801707923412323,
|
|
"epoch": 0.21,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 7.95e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 351955.0,
|
|
"reward": -9.899999618530273,
|
|
"reward_std": 0.0,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -10.0,
|
|
"rewards/supergames_reward/std": 0.0,
|
|
"step": 42,
|
|
"step_time": 5.711630532023264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 308.0,
|
|
"completions/max_terminated_length": 308.0,
|
|
"completions/mean_length": 247.375,
|
|
"completions/mean_terminated_length": 247.375,
|
|
"completions/min_length": 116.0,
|
|
"completions/min_terminated_length": 116.0,
|
|
"entropy": 0.3185043931007385,
|
|
"epoch": 0.215,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.168762445449829,
|
|
"learning_rate": 7.9e-07,
|
|
"loss": 0.1970764398574829,
|
|
"num_tokens": 363278.0,
|
|
"reward": -53.89249801635742,
|
|
"reward_std": 63.02619934082031,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -53.99250030517578,
|
|
"rewards/supergames_reward/std": 63.02619934082031,
|
|
"step": 43,
|
|
"step_time": 11.661934906995157
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 330.0,
|
|
"completions/max_terminated_length": 330.0,
|
|
"completions/mean_length": 212.5,
|
|
"completions/mean_terminated_length": 212.5,
|
|
"completions/min_length": 106.0,
|
|
"completions/min_terminated_length": 106.0,
|
|
"entropy": 0.4446268379688263,
|
|
"epoch": 0.22,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.305844306945801,
|
|
"learning_rate": 7.85e-07,
|
|
"loss": -0.06735197454690933,
|
|
"num_tokens": 374306.0,
|
|
"reward": 18.059999465942383,
|
|
"reward_std": 98.9417953491211,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 17.959999084472656,
|
|
"rewards/supergames_reward/std": 98.9417953491211,
|
|
"step": 44,
|
|
"step_time": 12.492453911982011
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.0,
|
|
"completions/max_terminated_length": 316.0,
|
|
"completions/mean_length": 183.0,
|
|
"completions/mean_terminated_length": 183.0,
|
|
"completions/min_length": 103.0,
|
|
"completions/min_terminated_length": 103.0,
|
|
"entropy": 0.5269614458084106,
|
|
"epoch": 0.225,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.8006370067596436,
|
|
"learning_rate": 7.799999999999999e-07,
|
|
"loss": -0.08870815485715866,
|
|
"num_tokens": 382210.0,
|
|
"reward": -54.928749084472656,
|
|
"reward_std": 72.5127944946289,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -55.02874755859375,
|
|
"rewards/supergames_reward/std": 72.5127944946289,
|
|
"step": 45,
|
|
"step_time": 11.7636467939883
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 244.0,
|
|
"completions/max_terminated_length": 244.0,
|
|
"completions/mean_length": 136.625,
|
|
"completions/mean_terminated_length": 136.625,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 0.34820204973220825,
|
|
"epoch": 0.23,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.911736011505127,
|
|
"learning_rate": 7.75e-07,
|
|
"loss": 0.04397330805659294,
|
|
"num_tokens": 391583.0,
|
|
"reward": -19.10375213623047,
|
|
"reward_std": 23.967702865600586,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -19.203750610351562,
|
|
"rewards/supergames_reward/std": 23.967702865600586,
|
|
"step": 46,
|
|
"step_time": 10.119426151999505
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 218.0,
|
|
"completions/max_terminated_length": 218.0,
|
|
"completions/mean_length": 160.375,
|
|
"completions/mean_terminated_length": 160.375,
|
|
"completions/min_length": 93.0,
|
|
"completions/min_terminated_length": 93.0,
|
|
"entropy": 0.611914336681366,
|
|
"epoch": 0.235,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.571913242340088,
|
|
"learning_rate": 7.699999999999999e-07,
|
|
"loss": -0.044293053448200226,
|
|
"num_tokens": 399290.0,
|
|
"reward": -25.186250686645508,
|
|
"reward_std": 71.46305084228516,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -25.286251068115234,
|
|
"rewards/supergames_reward/std": 71.46305084228516,
|
|
"step": 47,
|
|
"step_time": 8.563868903991533
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 167.0,
|
|
"completions/max_terminated_length": 167.0,
|
|
"completions/mean_length": 111.125,
|
|
"completions/mean_terminated_length": 111.125,
|
|
"completions/min_length": 56.0,
|
|
"completions/min_terminated_length": 56.0,
|
|
"entropy": 0.46585777401924133,
|
|
"epoch": 0.24,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.654109954833984,
|
|
"learning_rate": 7.65e-07,
|
|
"loss": -0.035151124000549316,
|
|
"num_tokens": 404707.0,
|
|
"reward": -0.408750057220459,
|
|
"reward_std": 31.939748764038086,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -0.5087499618530273,
|
|
"rewards/supergames_reward/std": 31.93975257873535,
|
|
"step": 48,
|
|
"step_time": 6.438482580007985
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 306.0,
|
|
"completions/max_terminated_length": 306.0,
|
|
"completions/mean_length": 216.625,
|
|
"completions/mean_terminated_length": 216.625,
|
|
"completions/min_length": 132.0,
|
|
"completions/min_terminated_length": 132.0,
|
|
"entropy": 0.33054471015930176,
|
|
"epoch": 0.245,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.700303792953491,
|
|
"learning_rate": 7.599999999999999e-07,
|
|
"loss": -0.04393656551837921,
|
|
"num_tokens": 414736.0,
|
|
"reward": -97.26000213623047,
|
|
"reward_std": 1.6294406652450562,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -97.36000061035156,
|
|
"rewards/supergames_reward/std": 1.629441738128662,
|
|
"step": 49,
|
|
"step_time": 11.636040106008295
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 378.0,
|
|
"completions/max_terminated_length": 378.0,
|
|
"completions/mean_length": 218.375,
|
|
"completions/mean_terminated_length": 218.375,
|
|
"completions/min_length": 104.0,
|
|
"completions/min_terminated_length": 104.0,
|
|
"entropy": 0.48711177706718445,
|
|
"epoch": 0.25,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.9650564193725586,
|
|
"learning_rate": 7.55e-07,
|
|
"loss": -0.3529520332813263,
|
|
"num_tokens": 424795.0,
|
|
"reward": 47.408748626708984,
|
|
"reward_std": 72.72083282470703,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 47.30875015258789,
|
|
"rewards/supergames_reward/std": 72.72083282470703,
|
|
"step": 50,
|
|
"step_time": 14.258096842997475
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 492.0,
|
|
"completions/mean_length": 267.625,
|
|
"completions/mean_terminated_length": 232.71429443359375,
|
|
"completions/min_length": 90.0,
|
|
"completions/min_terminated_length": 90.0,
|
|
"entropy": 0.4482860863208771,
|
|
"epoch": 0.255,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.2998197078704834,
|
|
"learning_rate": 7.5e-07,
|
|
"loss": 0.2493654191493988,
|
|
"num_tokens": 436288.0,
|
|
"reward": 50.61125183105469,
|
|
"reward_std": 75.3349838256836,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 50.53624725341797,
|
|
"rewards/supergames_reward/std": 75.27783966064453,
|
|
"step": 51,
|
|
"step_time": 19.20827590499539
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 176.0,
|
|
"completions/max_terminated_length": 176.0,
|
|
"completions/mean_length": 144.5,
|
|
"completions/mean_terminated_length": 144.5,
|
|
"completions/min_length": 99.0,
|
|
"completions/min_terminated_length": 99.0,
|
|
"entropy": 0.5378735661506653,
|
|
"epoch": 0.26,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.114461898803711,
|
|
"learning_rate": 7.45e-07,
|
|
"loss": 0.0036597400903701782,
|
|
"num_tokens": 441980.0,
|
|
"reward": -15.20625114440918,
|
|
"reward_std": 45.35526657104492,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -15.28125,
|
|
"rewards/supergames_reward/std": 45.301815032958984,
|
|
"step": 52,
|
|
"step_time": 6.660627231001854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 136.0,
|
|
"completions/max_terminated_length": 136.0,
|
|
"completions/mean_length": 119.0,
|
|
"completions/mean_terminated_length": 119.0,
|
|
"completions/min_length": 105.0,
|
|
"completions/min_terminated_length": 105.0,
|
|
"entropy": 0.41796669363975525,
|
|
"epoch": 0.265,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.059505462646484,
|
|
"learning_rate": 7.4e-07,
|
|
"loss": -0.014973883517086506,
|
|
"num_tokens": 447468.0,
|
|
"reward": 15.170000076293945,
|
|
"reward_std": 46.658329010009766,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 15.070000648498535,
|
|
"rewards/supergames_reward/std": 46.658329010009766,
|
|
"step": 53,
|
|
"step_time": 5.271571868011961
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 259.0,
|
|
"completions/max_terminated_length": 259.0,
|
|
"completions/mean_length": 191.125,
|
|
"completions/mean_terminated_length": 191.125,
|
|
"completions/min_length": 91.0,
|
|
"completions/min_terminated_length": 91.0,
|
|
"entropy": 0.45421987771987915,
|
|
"epoch": 0.27,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.4454867839813232,
|
|
"learning_rate": 7.35e-07,
|
|
"loss": -0.1128973588347435,
|
|
"num_tokens": 457301.0,
|
|
"reward": 38.89875030517578,
|
|
"reward_std": 62.1772575378418,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 38.79874801635742,
|
|
"rewards/supergames_reward/std": 62.1772575378418,
|
|
"step": 54,
|
|
"step_time": 10.524528659996577
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 319.0,
|
|
"completions/max_terminated_length": 319.0,
|
|
"completions/mean_length": 233.375,
|
|
"completions/mean_terminated_length": 233.375,
|
|
"completions/min_length": 136.0,
|
|
"completions/min_terminated_length": 136.0,
|
|
"entropy": 0.4449213445186615,
|
|
"epoch": 0.275,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.2065398693084717,
|
|
"learning_rate": 7.3e-07,
|
|
"loss": 0.0016277075046673417,
|
|
"num_tokens": 467520.0,
|
|
"reward": 27.372499465942383,
|
|
"reward_std": 81.23126983642578,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 27.27250099182129,
|
|
"rewards/supergames_reward/std": 81.23126983642578,
|
|
"step": 55,
|
|
"step_time": 12.592280682991259
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 357.0,
|
|
"completions/mean_length": 261.0,
|
|
"completions/mean_terminated_length": 225.1428680419922,
|
|
"completions/min_length": 95.0,
|
|
"completions/min_terminated_length": 95.0,
|
|
"entropy": 0.506920337677002,
|
|
"epoch": 0.28,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.058388710021973,
|
|
"learning_rate": 7.249999999999999e-07,
|
|
"loss": 0.3289242386817932,
|
|
"num_tokens": 476056.0,
|
|
"reward": 47.05249786376953,
|
|
"reward_std": 80.97753143310547,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 46.977500915527344,
|
|
"rewards/supergames_reward/std": 80.9256362915039,
|
|
"step": 56,
|
|
"step_time": 18.845177220006008
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 250.0,
|
|
"completions/max_terminated_length": 250.0,
|
|
"completions/mean_length": 192.75,
|
|
"completions/mean_terminated_length": 192.75,
|
|
"completions/min_length": 122.0,
|
|
"completions/min_terminated_length": 122.0,
|
|
"entropy": 0.5197336077690125,
|
|
"epoch": 0.285,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.230815410614014,
|
|
"learning_rate": 7.2e-07,
|
|
"loss": -0.15186546742916107,
|
|
"num_tokens": 484014.0,
|
|
"reward": -64.5050048828125,
|
|
"reward_std": 50.35145568847656,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -64.60499572753906,
|
|
"rewards/supergames_reward/std": 50.3514518737793,
|
|
"step": 57,
|
|
"step_time": 9.576232638006331
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 324.0,
|
|
"completions/max_terminated_length": 324.0,
|
|
"completions/mean_length": 249.0,
|
|
"completions/mean_terminated_length": 249.0,
|
|
"completions/min_length": 129.0,
|
|
"completions/min_terminated_length": 129.0,
|
|
"entropy": 0.42913514375686646,
|
|
"epoch": 0.29,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.8544528484344482,
|
|
"learning_rate": 7.149999999999999e-07,
|
|
"loss": -0.056405920535326004,
|
|
"num_tokens": 494390.0,
|
|
"reward": 40.123748779296875,
|
|
"reward_std": 84.164794921875,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 40.02375030517578,
|
|
"rewards/supergames_reward/std": 84.164794921875,
|
|
"step": 58,
|
|
"step_time": 12.385353341000155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 138.0,
|
|
"completions/max_terminated_length": 138.0,
|
|
"completions/mean_length": 107.25,
|
|
"completions/mean_terminated_length": 107.25,
|
|
"completions/min_length": 81.0,
|
|
"completions/min_terminated_length": 81.0,
|
|
"entropy": 0.34386488795280457,
|
|
"epoch": 0.295,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.6915154457092285,
|
|
"learning_rate": 7.1e-07,
|
|
"loss": 0.0865321159362793,
|
|
"num_tokens": 499816.0,
|
|
"reward": -8.244999885559082,
|
|
"reward_std": 4.681046962738037,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -8.345000267028809,
|
|
"rewards/supergames_reward/std": 4.681046962738037,
|
|
"step": 59,
|
|
"step_time": 5.42325339600211
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 187.0,
|
|
"completions/max_terminated_length": 187.0,
|
|
"completions/mean_length": 127.25,
|
|
"completions/mean_terminated_length": 127.25,
|
|
"completions/min_length": 97.0,
|
|
"completions/min_terminated_length": 97.0,
|
|
"entropy": 0.5430045127868652,
|
|
"epoch": 0.3,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.635311126708984,
|
|
"learning_rate": 7.049999999999999e-07,
|
|
"loss": -0.05412375554442406,
|
|
"num_tokens": 505410.0,
|
|
"reward": 20.005001068115234,
|
|
"reward_std": 55.156005859375,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 19.904998779296875,
|
|
"rewards/supergames_reward/std": 55.156005859375,
|
|
"step": 60,
|
|
"step_time": 7.068010902003152
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 286.0,
|
|
"completions/max_terminated_length": 286.0,
|
|
"completions/mean_length": 192.125,
|
|
"completions/mean_terminated_length": 192.125,
|
|
"completions/min_length": 95.0,
|
|
"completions/min_terminated_length": 95.0,
|
|
"entropy": 0.4213869869709015,
|
|
"epoch": 0.305,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.4664618968963623,
|
|
"learning_rate": 7e-07,
|
|
"loss": -0.0495159812271595,
|
|
"num_tokens": 516283.0,
|
|
"reward": -47.212501525878906,
|
|
"reward_std": 91.2253646850586,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -47.3125,
|
|
"rewards/supergames_reward/std": 91.2253646850586,
|
|
"step": 61,
|
|
"step_time": 11.176304235996213
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 324.0,
|
|
"completions/max_terminated_length": 324.0,
|
|
"completions/mean_length": 171.0,
|
|
"completions/mean_terminated_length": 171.0,
|
|
"completions/min_length": 92.0,
|
|
"completions/min_terminated_length": 92.0,
|
|
"entropy": 0.4626811146736145,
|
|
"epoch": 0.31,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.949252128601074,
|
|
"learning_rate": 6.949999999999999e-07,
|
|
"loss": 0.192047581076622,
|
|
"num_tokens": 522251.0,
|
|
"reward": -15.092500686645508,
|
|
"reward_std": 49.82962417602539,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -15.167499542236328,
|
|
"rewards/supergames_reward/std": 49.78091049194336,
|
|
"step": 62,
|
|
"step_time": 12.075224861997413
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 429.0,
|
|
"completions/mean_length": 283.5,
|
|
"completions/mean_terminated_length": 250.85714721679688,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 0.4814216196537018,
|
|
"epoch": 0.315,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.7918753623962402,
|
|
"learning_rate": 6.9e-07,
|
|
"loss": 0.15706156194210052,
|
|
"num_tokens": 533815.0,
|
|
"reward": -20.86625099182129,
|
|
"reward_std": 85.99250030517578,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -20.94124984741211,
|
|
"rewards/supergames_reward/std": 85.96620178222656,
|
|
"step": 63,
|
|
"step_time": 19.448832260008203
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 189.0,
|
|
"completions/max_terminated_length": 189.0,
|
|
"completions/mean_length": 131.25,
|
|
"completions/mean_terminated_length": 131.25,
|
|
"completions/min_length": 106.0,
|
|
"completions/min_terminated_length": 106.0,
|
|
"entropy": 0.4929414987564087,
|
|
"epoch": 0.32,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.774101734161377,
|
|
"learning_rate": 6.85e-07,
|
|
"loss": -0.060691747814416885,
|
|
"num_tokens": 539465.0,
|
|
"reward": -21.0674991607666,
|
|
"reward_std": 6.892730236053467,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -21.167499542236328,
|
|
"rewards/supergames_reward/std": 6.892730236053467,
|
|
"step": 64,
|
|
"step_time": 7.141569407976931
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 454.0,
|
|
"completions/max_terminated_length": 454.0,
|
|
"completions/mean_length": 281.375,
|
|
"completions/mean_terminated_length": 281.375,
|
|
"completions/min_length": 107.0,
|
|
"completions/min_terminated_length": 107.0,
|
|
"entropy": 0.45866572856903076,
|
|
"epoch": 0.325,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.092717409133911,
|
|
"learning_rate": 6.800000000000001e-07,
|
|
"loss": 0.11694959551095963,
|
|
"num_tokens": 551012.0,
|
|
"reward": 49.938751220703125,
|
|
"reward_std": 79.1865005493164,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 49.8387451171875,
|
|
"rewards/supergames_reward/std": 79.1865005493164,
|
|
"step": 65,
|
|
"step_time": 17.06170882002334
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 203.0,
|
|
"completions/max_terminated_length": 203.0,
|
|
"completions/mean_length": 128.625,
|
|
"completions/mean_terminated_length": 128.625,
|
|
"completions/min_length": 88.0,
|
|
"completions/min_terminated_length": 88.0,
|
|
"entropy": 0.5005782246589661,
|
|
"epoch": 0.33,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.77616024017334,
|
|
"learning_rate": 6.75e-07,
|
|
"loss": 0.11242837458848953,
|
|
"num_tokens": 556577.0,
|
|
"reward": 48.875,
|
|
"reward_std": 49.13539123535156,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 48.775001525878906,
|
|
"rewards/supergames_reward/std": 49.13539123535156,
|
|
"step": 66,
|
|
"step_time": 7.695410129002994
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 436.0,
|
|
"completions/max_terminated_length": 436.0,
|
|
"completions/mean_length": 281.875,
|
|
"completions/mean_terminated_length": 281.875,
|
|
"completions/min_length": 133.0,
|
|
"completions/min_terminated_length": 133.0,
|
|
"entropy": 0.507097065448761,
|
|
"epoch": 0.335,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.911184787750244,
|
|
"learning_rate": 6.7e-07,
|
|
"loss": -0.08151137083768845,
|
|
"num_tokens": 567120.0,
|
|
"reward": -12.886249542236328,
|
|
"reward_std": 84.04269409179688,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -12.986251831054688,
|
|
"rewards/supergames_reward/std": 84.04269409179688,
|
|
"step": 67,
|
|
"step_time": 16.409127942984924
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 501.0,
|
|
"completions/mean_length": 318.375,
|
|
"completions/mean_terminated_length": 290.71429443359375,
|
|
"completions/min_length": 65.0,
|
|
"completions/min_terminated_length": 65.0,
|
|
"entropy": 0.486402690410614,
|
|
"epoch": 0.34,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.9461607933044434,
|
|
"learning_rate": 6.65e-07,
|
|
"loss": 0.2442178875207901,
|
|
"num_tokens": 578995.0,
|
|
"reward": 2.8387489318847656,
|
|
"reward_std": 98.47579956054688,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 2.7637500762939453,
|
|
"rewards/supergames_reward/std": 98.44595336914062,
|
|
"step": 68,
|
|
"step_time": 19.32364999302081
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 291.0,
|
|
"completions/max_terminated_length": 291.0,
|
|
"completions/mean_length": 219.625,
|
|
"completions/mean_terminated_length": 219.625,
|
|
"completions/min_length": 117.0,
|
|
"completions/min_terminated_length": 117.0,
|
|
"entropy": 0.37856727838516235,
|
|
"epoch": 0.345,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.0204732418060303,
|
|
"learning_rate": 6.6e-07,
|
|
"loss": -0.08764916658401489,
|
|
"num_tokens": 589072.0,
|
|
"reward": -54.18375015258789,
|
|
"reward_std": 71.45012664794922,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -54.283748626708984,
|
|
"rewards/supergames_reward/std": 71.45013427734375,
|
|
"step": 69,
|
|
"step_time": 11.126611230982235
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 305.0,
|
|
"completions/mean_length": 254.75,
|
|
"completions/mean_terminated_length": 218.00001525878906,
|
|
"completions/min_length": 104.0,
|
|
"completions/min_terminated_length": 104.0,
|
|
"entropy": 0.46423614025115967,
|
|
"epoch": 0.35,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.0140154361724854,
|
|
"learning_rate": 6.55e-07,
|
|
"loss": 0.26797160506248474,
|
|
"num_tokens": 600438.0,
|
|
"reward": -32.04875183105469,
|
|
"reward_std": 69.86957550048828,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -32.12375259399414,
|
|
"rewards/supergames_reward/std": 69.84178161621094,
|
|
"step": 70,
|
|
"step_time": 19.12617294798838
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 372.0,
|
|
"completions/max_terminated_length": 372.0,
|
|
"completions/mean_length": 194.125,
|
|
"completions/mean_terminated_length": 194.125,
|
|
"completions/min_length": 94.0,
|
|
"completions/min_terminated_length": 94.0,
|
|
"entropy": 0.3427293300628662,
|
|
"epoch": 0.355,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.781383991241455,
|
|
"learning_rate": 6.5e-07,
|
|
"loss": 0.354524701833725,
|
|
"num_tokens": 610327.0,
|
|
"reward": -6.913749694824219,
|
|
"reward_std": 87.56071472167969,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -6.988750457763672,
|
|
"rewards/supergames_reward/std": 87.53033447265625,
|
|
"step": 71,
|
|
"step_time": 14.041668199002743
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 197.0,
|
|
"completions/max_terminated_length": 197.0,
|
|
"completions/mean_length": 124.75,
|
|
"completions/mean_terminated_length": 124.75,
|
|
"completions/min_length": 93.0,
|
|
"completions/min_terminated_length": 93.0,
|
|
"entropy": 0.4455118477344513,
|
|
"epoch": 0.36,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.726032733917236,
|
|
"learning_rate": 6.45e-07,
|
|
"loss": 0.18206289410591125,
|
|
"num_tokens": 615853.0,
|
|
"reward": 41.95124816894531,
|
|
"reward_std": 60.640811920166016,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 41.85124969482422,
|
|
"rewards/supergames_reward/std": 60.640811920166016,
|
|
"step": 72,
|
|
"step_time": 7.416493425989756
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 323.0,
|
|
"completions/max_terminated_length": 323.0,
|
|
"completions/mean_length": 147.375,
|
|
"completions/mean_terminated_length": 147.375,
|
|
"completions/min_length": 94.0,
|
|
"completions/min_terminated_length": 94.0,
|
|
"entropy": 0.480733186006546,
|
|
"epoch": 0.365,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.853621482849121,
|
|
"learning_rate": 6.4e-07,
|
|
"loss": 0.04558124020695686,
|
|
"num_tokens": 621584.0,
|
|
"reward": 14.884998321533203,
|
|
"reward_std": 46.25857162475586,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 14.78499984741211,
|
|
"rewards/supergames_reward/std": 46.25857162475586,
|
|
"step": 73,
|
|
"step_time": 11.95379540900467
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 283.0,
|
|
"completions/max_terminated_length": 283.0,
|
|
"completions/mean_length": 188.125,
|
|
"completions/mean_terminated_length": 188.125,
|
|
"completions/min_length": 116.0,
|
|
"completions/min_terminated_length": 116.0,
|
|
"entropy": 0.5526013374328613,
|
|
"epoch": 0.37,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.729883909225464,
|
|
"learning_rate": 6.35e-07,
|
|
"loss": 0.06268303096294403,
|
|
"num_tokens": 629545.0,
|
|
"reward": -68.05750274658203,
|
|
"reward_std": 59.44808578491211,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -68.15750122070312,
|
|
"rewards/supergames_reward/std": 59.44808578491211,
|
|
"step": 74,
|
|
"step_time": 10.64124580900534
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 141.0,
|
|
"completions/max_terminated_length": 141.0,
|
|
"completions/mean_length": 117.375,
|
|
"completions/mean_terminated_length": 117.375,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"entropy": 0.3744319677352905,
|
|
"epoch": 0.375,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.775154113769531,
|
|
"learning_rate": 6.3e-07,
|
|
"loss": -0.023284845054149628,
|
|
"num_tokens": 635052.0,
|
|
"reward": 3.9374990463256836,
|
|
"reward_std": 61.85771179199219,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 3.862499237060547,
|
|
"rewards/supergames_reward/std": 61.80967712402344,
|
|
"step": 75,
|
|
"step_time": 5.432648951013107
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 268.0,
|
|
"completions/max_terminated_length": 268.0,
|
|
"completions/mean_length": 157.0,
|
|
"completions/mean_terminated_length": 157.0,
|
|
"completions/min_length": 104.0,
|
|
"completions/min_terminated_length": 104.0,
|
|
"entropy": 0.4822140336036682,
|
|
"epoch": 0.38,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.715704917907715,
|
|
"learning_rate": 6.249999999999999e-07,
|
|
"loss": -0.04918142035603523,
|
|
"num_tokens": 644644.0,
|
|
"reward": 48.64125061035156,
|
|
"reward_std": 78.0205078125,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 48.56624984741211,
|
|
"rewards/supergames_reward/std": 77.96605682373047,
|
|
"step": 76,
|
|
"step_time": 10.348935816989979
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 380.0,
|
|
"completions/max_terminated_length": 380.0,
|
|
"completions/mean_length": 210.875,
|
|
"completions/mean_terminated_length": 210.875,
|
|
"completions/min_length": 126.0,
|
|
"completions/min_terminated_length": 126.0,
|
|
"entropy": 0.45722317695617676,
|
|
"epoch": 0.385,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.6057567596435547,
|
|
"learning_rate": 6.2e-07,
|
|
"loss": 0.058352649211883545,
|
|
"num_tokens": 652771.0,
|
|
"reward": -8.283750534057617,
|
|
"reward_std": 77.68436431884766,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -8.383749008178711,
|
|
"rewards/supergames_reward/std": 77.68437194824219,
|
|
"step": 77,
|
|
"step_time": 14.142948755004909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 324.0,
|
|
"completions/max_terminated_length": 324.0,
|
|
"completions/mean_length": 188.25,
|
|
"completions/mean_terminated_length": 188.25,
|
|
"completions/min_length": 68.0,
|
|
"completions/min_terminated_length": 68.0,
|
|
"entropy": 0.33380764722824097,
|
|
"epoch": 0.39,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.2239863872528076,
|
|
"learning_rate": 6.149999999999999e-07,
|
|
"loss": -0.04592633992433548,
|
|
"num_tokens": 662653.0,
|
|
"reward": 11.153749465942383,
|
|
"reward_std": 89.88329315185547,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 11.053749084472656,
|
|
"rewards/supergames_reward/std": 89.88329315185547,
|
|
"step": 78,
|
|
"step_time": 12.277474621019792
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 143.0,
|
|
"completions/max_terminated_length": 143.0,
|
|
"completions/mean_length": 108.875,
|
|
"completions/mean_terminated_length": 108.875,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 0.3518812656402588,
|
|
"epoch": 0.395,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.967776298522949,
|
|
"learning_rate": 6.1e-07,
|
|
"loss": -0.10800496488809586,
|
|
"num_tokens": 668076.0,
|
|
"reward": 70.33250427246094,
|
|
"reward_std": 49.9145393371582,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 70.23249816894531,
|
|
"rewards/supergames_reward/std": 49.91453552246094,
|
|
"step": 79,
|
|
"step_time": 5.52714040101273
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 184.0,
|
|
"completions/max_terminated_length": 184.0,
|
|
"completions/mean_length": 152.125,
|
|
"completions/mean_terminated_length": 152.125,
|
|
"completions/min_length": 104.0,
|
|
"completions/min_terminated_length": 104.0,
|
|
"entropy": 0.4699896574020386,
|
|
"epoch": 0.4,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.561161994934082,
|
|
"learning_rate": 6.049999999999999e-07,
|
|
"loss": 0.09505834430456161,
|
|
"num_tokens": 675701.0,
|
|
"reward": -20.25625228881836,
|
|
"reward_std": 53.78542709350586,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -20.35624885559082,
|
|
"rewards/supergames_reward/std": 53.78542709350586,
|
|
"step": 80,
|
|
"step_time": 7.155081019998761
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 506.0,
|
|
"completions/mean_length": 288.875,
|
|
"completions/mean_terminated_length": 257.0,
|
|
"completions/min_length": 115.0,
|
|
"completions/min_terminated_length": 115.0,
|
|
"entropy": 0.4086494445800781,
|
|
"epoch": 0.405,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.3719611167907715,
|
|
"learning_rate": 6e-07,
|
|
"loss": 0.04727627709507942,
|
|
"num_tokens": 687332.0,
|
|
"reward": -37.20750045776367,
|
|
"reward_std": 91.7676010131836,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -37.25749969482422,
|
|
"rewards/supergames_reward/std": 91.72847747802734,
|
|
"step": 81,
|
|
"step_time": 19.231399144016905
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 376.0,
|
|
"completions/max_terminated_length": 376.0,
|
|
"completions/mean_length": 227.75,
|
|
"completions/mean_terminated_length": 227.75,
|
|
"completions/min_length": 39.0,
|
|
"completions/min_terminated_length": 39.0,
|
|
"entropy": 0.43233707547187805,
|
|
"epoch": 0.41,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.3391714096069336,
|
|
"learning_rate": 5.949999999999999e-07,
|
|
"loss": -0.25524550676345825,
|
|
"num_tokens": 698474.0,
|
|
"reward": -3.7287511825561523,
|
|
"reward_std": 91.25379943847656,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -3.828749656677246,
|
|
"rewards/supergames_reward/std": 91.25379943847656,
|
|
"step": 82,
|
|
"step_time": 14.329176368017215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.0,
|
|
"completions/max_terminated_length": 299.0,
|
|
"completions/mean_length": 178.125,
|
|
"completions/mean_terminated_length": 178.125,
|
|
"completions/min_length": 91.0,
|
|
"completions/min_terminated_length": 91.0,
|
|
"entropy": 0.4293636977672577,
|
|
"epoch": 0.415,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.7271742820739746,
|
|
"learning_rate": 5.9e-07,
|
|
"loss": 0.09273672848939896,
|
|
"num_tokens": 708243.0,
|
|
"reward": 12.802498817443848,
|
|
"reward_std": 86.23526000976562,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 12.702500343322754,
|
|
"rewards/supergames_reward/std": 86.23526000976562,
|
|
"step": 83,
|
|
"step_time": 11.419686011999147
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 425.0,
|
|
"completions/max_terminated_length": 425.0,
|
|
"completions/mean_length": 255.375,
|
|
"completions/mean_terminated_length": 255.375,
|
|
"completions/min_length": 172.0,
|
|
"completions/min_terminated_length": 172.0,
|
|
"entropy": 0.38068246841430664,
|
|
"epoch": 0.42,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.6244184970855713,
|
|
"learning_rate": 5.849999999999999e-07,
|
|
"loss": 0.02006213366985321,
|
|
"num_tokens": 718606.0,
|
|
"reward": 37.803749084472656,
|
|
"reward_std": 57.58624267578125,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 37.70375061035156,
|
|
"rewards/supergames_reward/std": 57.58624267578125,
|
|
"step": 84,
|
|
"step_time": 15.975198492989875
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 116.0,
|
|
"completions/max_terminated_length": 116.0,
|
|
"completions/mean_length": 108.125,
|
|
"completions/mean_terminated_length": 108.125,
|
|
"completions/min_length": 87.0,
|
|
"completions/min_terminated_length": 87.0,
|
|
"entropy": 0.339572012424469,
|
|
"epoch": 0.425,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.754759311676025,
|
|
"learning_rate": 5.8e-07,
|
|
"loss": -0.011015941388905048,
|
|
"num_tokens": 723983.0,
|
|
"reward": -3.8212504386901855,
|
|
"reward_std": 61.624786376953125,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -3.921250820159912,
|
|
"rewards/supergames_reward/std": 61.624786376953125,
|
|
"step": 85,
|
|
"step_time": 4.542777584982105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 447.0,
|
|
"completions/max_terminated_length": 447.0,
|
|
"completions/mean_length": 291.375,
|
|
"completions/mean_terminated_length": 291.375,
|
|
"completions/min_length": 163.0,
|
|
"completions/min_terminated_length": 163.0,
|
|
"entropy": 0.5846173763275146,
|
|
"epoch": 0.43,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.55724835395813,
|
|
"learning_rate": 5.749999999999999e-07,
|
|
"loss": -0.04981120675802231,
|
|
"num_tokens": 735642.0,
|
|
"reward": 50.07499694824219,
|
|
"reward_std": 92.6283187866211,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 50.0,
|
|
"rewards/supergames_reward/std": 92.5820083618164,
|
|
"step": 86,
|
|
"step_time": 16.777178087009815
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 232.0,
|
|
"completions/max_terminated_length": 232.0,
|
|
"completions/mean_length": 192.0,
|
|
"completions/mean_terminated_length": 192.0,
|
|
"completions/min_length": 139.0,
|
|
"completions/min_terminated_length": 139.0,
|
|
"entropy": 0.5139474272727966,
|
|
"epoch": 0.435,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.8860411643981934,
|
|
"learning_rate": 5.699999999999999e-07,
|
|
"loss": -0.02194221317768097,
|
|
"num_tokens": 743602.0,
|
|
"reward": -52.522499084472656,
|
|
"reward_std": 73.8686294555664,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -52.62249755859375,
|
|
"rewards/supergames_reward/std": 73.86863708496094,
|
|
"step": 87,
|
|
"step_time": 8.772893911984283
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 266.0,
|
|
"completions/max_terminated_length": 266.0,
|
|
"completions/mean_length": 146.875,
|
|
"completions/mean_terminated_length": 146.875,
|
|
"completions/min_length": 89.0,
|
|
"completions/min_terminated_length": 89.0,
|
|
"entropy": 0.3428497910499573,
|
|
"epoch": 0.44,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.509483337402344,
|
|
"learning_rate": 5.649999999999999e-07,
|
|
"loss": 0.19857533276081085,
|
|
"num_tokens": 753065.0,
|
|
"reward": 63.63874816894531,
|
|
"reward_std": 67.54963684082031,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 63.53874969482422,
|
|
"rewards/supergames_reward/std": 67.54963684082031,
|
|
"step": 88,
|
|
"step_time": 10.230529835011112
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 446.0,
|
|
"completions/max_terminated_length": 446.0,
|
|
"completions/mean_length": 270.625,
|
|
"completions/mean_terminated_length": 270.625,
|
|
"completions/min_length": 103.0,
|
|
"completions/min_terminated_length": 103.0,
|
|
"entropy": 0.4417419731616974,
|
|
"epoch": 0.445,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.973951816558838,
|
|
"learning_rate": 5.6e-07,
|
|
"loss": -0.00031440958264283836,
|
|
"num_tokens": 764534.0,
|
|
"reward": -9.736251831054688,
|
|
"reward_std": 92.25879669189453,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -9.836250305175781,
|
|
"rewards/supergames_reward/std": 92.25880432128906,
|
|
"step": 89,
|
|
"step_time": 17.40752330099349
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 398.0,
|
|
"completions/max_terminated_length": 398.0,
|
|
"completions/mean_length": 227.375,
|
|
"completions/mean_terminated_length": 227.375,
|
|
"completions/min_length": 98.0,
|
|
"completions/min_terminated_length": 98.0,
|
|
"entropy": 0.4732024073600769,
|
|
"epoch": 0.45,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.2560815811157227,
|
|
"learning_rate": 5.55e-07,
|
|
"loss": 0.03820464387536049,
|
|
"num_tokens": 772809.0,
|
|
"reward": 5.982499599456787,
|
|
"reward_std": 58.833797454833984,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 5.8824992179870605,
|
|
"rewards/supergames_reward/std": 58.83380126953125,
|
|
"step": 90,
|
|
"step_time": 15.007269965979503
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.0,
|
|
"completions/max_terminated_length": 269.0,
|
|
"completions/mean_length": 207.5,
|
|
"completions/mean_terminated_length": 207.5,
|
|
"completions/min_length": 92.0,
|
|
"completions/min_terminated_length": 92.0,
|
|
"entropy": 0.42038559913635254,
|
|
"epoch": 0.455,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.3649117946624756,
|
|
"learning_rate": 5.5e-07,
|
|
"loss": 0.1510048508644104,
|
|
"num_tokens": 782805.0,
|
|
"reward": -5.61500358581543,
|
|
"reward_std": 82.67152404785156,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -5.714998245239258,
|
|
"rewards/supergames_reward/std": 82.67152404785156,
|
|
"step": 91,
|
|
"step_time": 10.386137172987219
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 416.0,
|
|
"completions/max_terminated_length": 416.0,
|
|
"completions/mean_length": 218.375,
|
|
"completions/mean_terminated_length": 218.375,
|
|
"completions/min_length": 95.0,
|
|
"completions/min_terminated_length": 95.0,
|
|
"entropy": 0.3971734642982483,
|
|
"epoch": 0.46,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.433772563934326,
|
|
"learning_rate": 5.45e-07,
|
|
"loss": -0.3027213513851166,
|
|
"num_tokens": 793840.0,
|
|
"reward": -29.06500244140625,
|
|
"reward_std": 91.68325805664062,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -29.16499900817871,
|
|
"rewards/supergames_reward/std": 91.68325805664062,
|
|
"step": 92,
|
|
"step_time": 15.790611553995404
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 345.0,
|
|
"completions/max_terminated_length": 345.0,
|
|
"completions/mean_length": 206.625,
|
|
"completions/mean_terminated_length": 206.625,
|
|
"completions/min_length": 90.0,
|
|
"completions/min_terminated_length": 90.0,
|
|
"entropy": 0.5906907320022583,
|
|
"epoch": 0.465,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.293088912963867,
|
|
"learning_rate": 5.4e-07,
|
|
"loss": 0.09760032594203949,
|
|
"num_tokens": 803821.0,
|
|
"reward": -72.73999786376953,
|
|
"reward_std": 50.72484588623047,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -72.81500244140625,
|
|
"rewards/supergames_reward/std": 50.7094841003418,
|
|
"step": 93,
|
|
"step_time": 13.043119941983605
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 143.0,
|
|
"completions/max_terminated_length": 143.0,
|
|
"completions/mean_length": 115.75,
|
|
"completions/mean_terminated_length": 115.75,
|
|
"completions/min_length": 80.0,
|
|
"completions/min_terminated_length": 80.0,
|
|
"entropy": 0.4233115315437317,
|
|
"epoch": 0.47,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.58886194229126,
|
|
"learning_rate": 5.35e-07,
|
|
"loss": -0.04040650278329849,
|
|
"num_tokens": 809299.0,
|
|
"reward": -3.3050003051757812,
|
|
"reward_std": 24.443714141845703,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -3.405000686645508,
|
|
"rewards/supergames_reward/std": 24.443714141845703,
|
|
"step": 94,
|
|
"step_time": 5.512874762003776
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 122.0,
|
|
"completions/max_terminated_length": 122.0,
|
|
"completions/mean_length": 103.125,
|
|
"completions/mean_terminated_length": 103.125,
|
|
"completions/min_length": 86.0,
|
|
"completions/min_terminated_length": 86.0,
|
|
"entropy": 0.4055764377117157,
|
|
"epoch": 0.475,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.981549263000488,
|
|
"learning_rate": 5.3e-07,
|
|
"loss": -0.07588938623666763,
|
|
"num_tokens": 814716.0,
|
|
"reward": 60.06999969482422,
|
|
"reward_std": 42.708473205566406,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 59.970001220703125,
|
|
"rewards/supergames_reward/std": 42.708473205566406,
|
|
"step": 95,
|
|
"step_time": 4.766389057011111
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 475.0,
|
|
"completions/max_terminated_length": 475.0,
|
|
"completions/mean_length": 218.375,
|
|
"completions/mean_terminated_length": 218.375,
|
|
"completions/min_length": 81.0,
|
|
"completions/min_terminated_length": 81.0,
|
|
"entropy": 0.408913254737854,
|
|
"epoch": 0.48,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.79402756690979,
|
|
"learning_rate": 5.25e-07,
|
|
"loss": -0.3087541460990906,
|
|
"num_tokens": 824791.0,
|
|
"reward": -15.042500495910645,
|
|
"reward_std": 84.37246704101562,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -15.142499923706055,
|
|
"rewards/supergames_reward/std": 84.37246704101562,
|
|
"step": 96,
|
|
"step_time": 17.779843941010768
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 154.0,
|
|
"completions/max_terminated_length": 154.0,
|
|
"completions/mean_length": 121.625,
|
|
"completions/mean_terminated_length": 121.625,
|
|
"completions/min_length": 80.0,
|
|
"completions/min_terminated_length": 80.0,
|
|
"entropy": 0.529015839099884,
|
|
"epoch": 0.485,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.142816066741943,
|
|
"learning_rate": 5.2e-07,
|
|
"loss": 0.07216037809848785,
|
|
"num_tokens": 830300.0,
|
|
"reward": -32.743751525878906,
|
|
"reward_std": 54.61775588989258,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -32.81875228881836,
|
|
"rewards/supergames_reward/std": 54.58256149291992,
|
|
"step": 97,
|
|
"step_time": 5.913989890017547
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 126.0,
|
|
"completions/max_terminated_length": 126.0,
|
|
"completions/mean_length": 95.625,
|
|
"completions/mean_terminated_length": 95.625,
|
|
"completions/min_length": 74.0,
|
|
"completions/min_terminated_length": 74.0,
|
|
"entropy": 0.33743274211883545,
|
|
"epoch": 0.49,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.197273254394531,
|
|
"learning_rate": 5.149999999999999e-07,
|
|
"loss": 0.07500407844781876,
|
|
"num_tokens": 835633.0,
|
|
"reward": 11.00999927520752,
|
|
"reward_std": 62.73848342895508,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 10.910000801086426,
|
|
"rewards/supergames_reward/std": 62.73848342895508,
|
|
"step": 98,
|
|
"step_time": 4.9155233050114475
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 149.0,
|
|
"completions/max_terminated_length": 149.0,
|
|
"completions/mean_length": 108.5,
|
|
"completions/mean_terminated_length": 108.5,
|
|
"completions/min_length": 86.0,
|
|
"completions/min_terminated_length": 86.0,
|
|
"entropy": 0.3106803297996521,
|
|
"epoch": 0.495,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.864002227783203,
|
|
"learning_rate": 5.1e-07,
|
|
"loss": -0.1206185445189476,
|
|
"num_tokens": 841045.0,
|
|
"reward": 14.238749504089355,
|
|
"reward_std": 51.77016830444336,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 14.138750076293945,
|
|
"rewards/supergames_reward/std": 51.77016830444336,
|
|
"step": 99,
|
|
"step_time": 5.7507751359953545
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 149.0,
|
|
"completions/max_terminated_length": 149.0,
|
|
"completions/mean_length": 97.125,
|
|
"completions/mean_terminated_length": 97.125,
|
|
"completions/min_length": 73.0,
|
|
"completions/min_terminated_length": 73.0,
|
|
"entropy": 0.35160186886787415,
|
|
"epoch": 0.5,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.915731906890869,
|
|
"learning_rate": 5.049999999999999e-07,
|
|
"loss": -0.07951541244983673,
|
|
"num_tokens": 846398.0,
|
|
"reward": 67.78125,
|
|
"reward_std": 38.15436553955078,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 67.68124389648438,
|
|
"rewards/supergames_reward/std": 38.15436553955078,
|
|
"step": 100,
|
|
"step_time": 5.994720440998208
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 320.0,
|
|
"completions/max_terminated_length": 320.0,
|
|
"completions/mean_length": 208.75,
|
|
"completions/mean_terminated_length": 208.75,
|
|
"completions/min_length": 95.0,
|
|
"completions/min_terminated_length": 95.0,
|
|
"entropy": 0.452095091342926,
|
|
"epoch": 0.505,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.065176963806152,
|
|
"learning_rate": 5e-07,
|
|
"loss": -0.1397111564874649,
|
|
"num_tokens": 857420.0,
|
|
"reward": -5.801251411437988,
|
|
"reward_std": 73.56117248535156,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -5.90125036239624,
|
|
"rewards/supergames_reward/std": 73.5611801147461,
|
|
"step": 101,
|
|
"step_time": 12.28178753197426
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 230.0,
|
|
"completions/max_terminated_length": 230.0,
|
|
"completions/mean_length": 171.375,
|
|
"completions/mean_terminated_length": 171.375,
|
|
"completions/min_length": 95.0,
|
|
"completions/min_terminated_length": 95.0,
|
|
"entropy": 0.4138447642326355,
|
|
"epoch": 0.51,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.998823404312134,
|
|
"learning_rate": 4.95e-07,
|
|
"loss": -0.08941879868507385,
|
|
"num_tokens": 867103.0,
|
|
"reward": 92.24749755859375,
|
|
"reward_std": 14.540005683898926,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 92.14749908447266,
|
|
"rewards/supergames_reward/std": 14.540006637573242,
|
|
"step": 102,
|
|
"step_time": 8.992682139010867
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 311.0,
|
|
"completions/max_terminated_length": 311.0,
|
|
"completions/mean_length": 163.0,
|
|
"completions/mean_terminated_length": 163.0,
|
|
"completions/min_length": 74.0,
|
|
"completions/min_terminated_length": 74.0,
|
|
"entropy": 0.4182547330856323,
|
|
"epoch": 0.515,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.419595241546631,
|
|
"learning_rate": 4.9e-07,
|
|
"loss": -0.19304415583610535,
|
|
"num_tokens": 876679.0,
|
|
"reward": 79.13249969482422,
|
|
"reward_std": 59.3050422668457,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 79.03250122070312,
|
|
"rewards/supergames_reward/std": 59.30504608154297,
|
|
"step": 103,
|
|
"step_time": 11.836970244999975
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 253.0,
|
|
"completions/max_terminated_length": 253.0,
|
|
"completions/mean_length": 155.625,
|
|
"completions/mean_terminated_length": 155.625,
|
|
"completions/min_length": 94.0,
|
|
"completions/min_terminated_length": 94.0,
|
|
"entropy": 0.49889081716537476,
|
|
"epoch": 0.52,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.463719367980957,
|
|
"learning_rate": 4.85e-07,
|
|
"loss": -0.04786720871925354,
|
|
"num_tokens": 884348.0,
|
|
"reward": 48.616249084472656,
|
|
"reward_std": 73.30741119384766,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 48.51625061035156,
|
|
"rewards/supergames_reward/std": 73.30741119384766,
|
|
"step": 104,
|
|
"step_time": 9.579507841990562
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 380.0,
|
|
"completions/max_terminated_length": 380.0,
|
|
"completions/mean_length": 269.5,
|
|
"completions/mean_terminated_length": 269.5,
|
|
"completions/min_length": 101.0,
|
|
"completions/min_terminated_length": 101.0,
|
|
"entropy": 0.5286482572555542,
|
|
"epoch": 0.525,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.993295431137085,
|
|
"learning_rate": 4.8e-07,
|
|
"loss": -0.08071611076593399,
|
|
"num_tokens": 894856.0,
|
|
"reward": 34.970001220703125,
|
|
"reward_std": 85.324462890625,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 34.869998931884766,
|
|
"rewards/supergames_reward/std": 85.324462890625,
|
|
"step": 105,
|
|
"step_time": 14.295730627985904
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 261.0,
|
|
"completions/max_terminated_length": 261.0,
|
|
"completions/mean_length": 203.25,
|
|
"completions/mean_terminated_length": 203.25,
|
|
"completions/min_length": 91.0,
|
|
"completions/min_terminated_length": 91.0,
|
|
"entropy": 0.6049264073371887,
|
|
"epoch": 0.53,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.787306785583496,
|
|
"learning_rate": 4.7499999999999995e-07,
|
|
"loss": 0.19928883016109467,
|
|
"num_tokens": 902922.0,
|
|
"reward": 32.44499969482422,
|
|
"reward_std": 70.4663314819336,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 32.369998931884766,
|
|
"rewards/supergames_reward/std": 70.41261291503906,
|
|
"step": 106,
|
|
"step_time": 9.84894504098338
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.0,
|
|
"completions/max_terminated_length": 279.0,
|
|
"completions/mean_length": 194.0,
|
|
"completions/mean_terminated_length": 194.0,
|
|
"completions/min_length": 109.0,
|
|
"completions/min_terminated_length": 109.0,
|
|
"entropy": 0.5115755200386047,
|
|
"epoch": 0.535,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.277207374572754,
|
|
"learning_rate": 4.6999999999999995e-07,
|
|
"loss": 0.09118300676345825,
|
|
"num_tokens": 910906.0,
|
|
"reward": 40.432498931884766,
|
|
"reward_std": 48.54548263549805,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 40.33250045776367,
|
|
"rewards/supergames_reward/std": 48.54548263549805,
|
|
"step": 107,
|
|
"step_time": 10.466603949986165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 332.0,
|
|
"completions/max_terminated_length": 332.0,
|
|
"completions/mean_length": 195.875,
|
|
"completions/mean_terminated_length": 195.875,
|
|
"completions/min_length": 87.0,
|
|
"completions/min_terminated_length": 87.0,
|
|
"entropy": 0.4192962944507599,
|
|
"epoch": 0.54,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.308061122894287,
|
|
"learning_rate": 4.65e-07,
|
|
"loss": 0.28064602613449097,
|
|
"num_tokens": 920745.0,
|
|
"reward": -21.982500076293945,
|
|
"reward_std": 66.31195068359375,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -22.08249855041504,
|
|
"rewards/supergames_reward/std": 66.31195068359375,
|
|
"step": 108,
|
|
"step_time": 12.595848233992001
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.0,
|
|
"completions/max_terminated_length": 269.0,
|
|
"completions/mean_length": 195.125,
|
|
"completions/mean_terminated_length": 195.125,
|
|
"completions/min_length": 111.0,
|
|
"completions/min_terminated_length": 111.0,
|
|
"entropy": 0.37523961067199707,
|
|
"epoch": 0.545,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.802706718444824,
|
|
"learning_rate": 4.6e-07,
|
|
"loss": -0.06614465266466141,
|
|
"num_tokens": 930618.0,
|
|
"reward": 71.75,
|
|
"reward_std": 51.14524459838867,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 71.6500015258789,
|
|
"rewards/supergames_reward/std": 51.1452522277832,
|
|
"step": 109,
|
|
"step_time": 10.33884758799104
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 331.0,
|
|
"completions/max_terminated_length": 331.0,
|
|
"completions/mean_length": 167.75,
|
|
"completions/mean_terminated_length": 167.75,
|
|
"completions/min_length": 89.0,
|
|
"completions/min_terminated_length": 89.0,
|
|
"entropy": 0.4339042007923126,
|
|
"epoch": 0.55,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.254003047943115,
|
|
"learning_rate": 4.55e-07,
|
|
"loss": 0.34405598044395447,
|
|
"num_tokens": 941256.0,
|
|
"reward": 99.21875,
|
|
"reward_std": 2.492553234100342,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 99.1187515258789,
|
|
"rewards/supergames_reward/std": 2.4925525188446045,
|
|
"step": 110,
|
|
"step_time": 12.657525141985388
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 285.0,
|
|
"completions/max_terminated_length": 285.0,
|
|
"completions/mean_length": 211.0,
|
|
"completions/mean_terminated_length": 211.0,
|
|
"completions/min_length": 112.0,
|
|
"completions/min_terminated_length": 112.0,
|
|
"entropy": 0.46809861063957214,
|
|
"epoch": 0.555,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.811166286468506,
|
|
"learning_rate": 4.5e-07,
|
|
"loss": 0.045113153755664825,
|
|
"num_tokens": 951296.0,
|
|
"reward": 55.27750015258789,
|
|
"reward_std": 47.90687561035156,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 55.1775016784668,
|
|
"rewards/supergames_reward/std": 47.9068717956543,
|
|
"step": 111,
|
|
"step_time": 10.91118389699841
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 145.0,
|
|
"completions/max_terminated_length": 145.0,
|
|
"completions/mean_length": 108.375,
|
|
"completions/mean_terminated_length": 108.375,
|
|
"completions/min_length": 86.0,
|
|
"completions/min_terminated_length": 86.0,
|
|
"entropy": 0.34181955456733704,
|
|
"epoch": 0.56,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.537296772003174,
|
|
"learning_rate": 4.45e-07,
|
|
"loss": -0.052413541823625565,
|
|
"num_tokens": 956675.0,
|
|
"reward": 47.56624984741211,
|
|
"reward_std": 49.185157775878906,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 47.466251373291016,
|
|
"rewards/supergames_reward/std": 49.18516159057617,
|
|
"step": 112,
|
|
"step_time": 5.552288047998445
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 245.0,
|
|
"completions/max_terminated_length": 245.0,
|
|
"completions/mean_length": 189.125,
|
|
"completions/mean_terminated_length": 189.125,
|
|
"completions/min_length": 102.0,
|
|
"completions/min_terminated_length": 102.0,
|
|
"entropy": 0.5034605860710144,
|
|
"epoch": 0.565,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.349876642227173,
|
|
"learning_rate": 4.3999999999999997e-07,
|
|
"loss": -0.08186715841293335,
|
|
"num_tokens": 964636.0,
|
|
"reward": 81.32374572753906,
|
|
"reward_std": 36.85074996948242,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 81.2237548828125,
|
|
"rewards/supergames_reward/std": 36.85074996948242,
|
|
"step": 113,
|
|
"step_time": 9.267611294984818
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 178.0,
|
|
"completions/max_terminated_length": 178.0,
|
|
"completions/mean_length": 111.25,
|
|
"completions/mean_terminated_length": 111.25,
|
|
"completions/min_length": 67.0,
|
|
"completions/min_terminated_length": 67.0,
|
|
"entropy": 0.3229002058506012,
|
|
"epoch": 0.57,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.245575428009033,
|
|
"learning_rate": 4.3499999999999996e-07,
|
|
"loss": 0.06544198095798492,
|
|
"num_tokens": 970070.0,
|
|
"reward": 63.1775016784668,
|
|
"reward_std": 52.30624008178711,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 63.07749938964844,
|
|
"rewards/supergames_reward/std": 52.30624008178711,
|
|
"step": 114,
|
|
"step_time": 6.731139309995342
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 224.0,
|
|
"completions/max_terminated_length": 224.0,
|
|
"completions/mean_length": 170.0,
|
|
"completions/mean_terminated_length": 170.0,
|
|
"completions/min_length": 94.0,
|
|
"completions/min_terminated_length": 94.0,
|
|
"entropy": 0.3274807929992676,
|
|
"epoch": 0.575,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.5544140338897705,
|
|
"learning_rate": 4.2999999999999996e-07,
|
|
"loss": -0.06953569501638412,
|
|
"num_tokens": 979702.0,
|
|
"reward": 25.62125015258789,
|
|
"reward_std": 64.52003479003906,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 25.521244049072266,
|
|
"rewards/supergames_reward/std": 64.52003479003906,
|
|
"step": 115,
|
|
"step_time": 8.702618776995223
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 427.0,
|
|
"completions/max_terminated_length": 427.0,
|
|
"completions/mean_length": 209.5,
|
|
"completions/mean_terminated_length": 209.5,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"entropy": 0.4970880150794983,
|
|
"epoch": 0.58,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.4932165145874023,
|
|
"learning_rate": 4.2499999999999995e-07,
|
|
"loss": -0.2281273603439331,
|
|
"num_tokens": 990690.0,
|
|
"reward": 50.54750061035156,
|
|
"reward_std": 70.5126953125,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 50.4474983215332,
|
|
"rewards/supergames_reward/std": 70.51270294189453,
|
|
"step": 116,
|
|
"step_time": 16.204386608995264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 228.0,
|
|
"completions/max_terminated_length": 228.0,
|
|
"completions/mean_length": 132.375,
|
|
"completions/mean_terminated_length": 132.375,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 0.4380227029323578,
|
|
"epoch": 0.585,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.089909076690674,
|
|
"learning_rate": 4.1999999999999995e-07,
|
|
"loss": 0.12121254950761795,
|
|
"num_tokens": 996333.0,
|
|
"reward": -15.333749771118164,
|
|
"reward_std": 47.200164794921875,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -15.433748245239258,
|
|
"rewards/supergames_reward/std": 47.200164794921875,
|
|
"step": 117,
|
|
"step_time": 8.546233230998041
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 323.0,
|
|
"completions/max_terminated_length": 323.0,
|
|
"completions/mean_length": 137.625,
|
|
"completions/mean_terminated_length": 137.625,
|
|
"completions/min_length": 90.0,
|
|
"completions/min_terminated_length": 90.0,
|
|
"entropy": 0.4177803695201874,
|
|
"epoch": 0.59,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.8712472915649414,
|
|
"learning_rate": 4.1499999999999994e-07,
|
|
"loss": 0.14192955195903778,
|
|
"num_tokens": 1002010.0,
|
|
"reward": -30.700000762939453,
|
|
"reward_std": 17.224035263061523,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -30.799999237060547,
|
|
"rewards/supergames_reward/std": 17.224035263061523,
|
|
"step": 118,
|
|
"step_time": 11.878434990998358
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 453.0,
|
|
"completions/max_terminated_length": 453.0,
|
|
"completions/mean_length": 238.125,
|
|
"completions/mean_terminated_length": 238.125,
|
|
"completions/min_length": 109.0,
|
|
"completions/min_terminated_length": 109.0,
|
|
"entropy": 0.5640217065811157,
|
|
"epoch": 0.595,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.064737319946289,
|
|
"learning_rate": 4.0999999999999994e-07,
|
|
"loss": -0.023621462285518646,
|
|
"num_tokens": 1012259.0,
|
|
"reward": 43.683746337890625,
|
|
"reward_std": 70.4210205078125,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 43.5837516784668,
|
|
"rewards/supergames_reward/std": 70.4210205078125,
|
|
"step": 119,
|
|
"step_time": 16.829514264973113
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 236.0,
|
|
"completions/max_terminated_length": 236.0,
|
|
"completions/mean_length": 193.75,
|
|
"completions/mean_terminated_length": 193.75,
|
|
"completions/min_length": 151.0,
|
|
"completions/min_terminated_length": 151.0,
|
|
"entropy": 0.5795091390609741,
|
|
"epoch": 0.6,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.423007965087891,
|
|
"learning_rate": 4.05e-07,
|
|
"loss": -0.0666111558675766,
|
|
"num_tokens": 1020241.0,
|
|
"reward": -85.86000061035156,
|
|
"reward_std": 27.4442081451416,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -85.90999603271484,
|
|
"rewards/supergames_reward/std": 27.41469955444336,
|
|
"step": 120,
|
|
"step_time": 8.95496519000153
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 331.0,
|
|
"completions/max_terminated_length": 331.0,
|
|
"completions/mean_length": 231.0,
|
|
"completions/mean_terminated_length": 231.0,
|
|
"completions/min_length": 165.0,
|
|
"completions/min_terminated_length": 165.0,
|
|
"entropy": 0.3767699897289276,
|
|
"epoch": 0.605,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.7434473037719727,
|
|
"learning_rate": 4e-07,
|
|
"loss": -0.1165132150053978,
|
|
"num_tokens": 1031465.0,
|
|
"reward": 61.45624923706055,
|
|
"reward_std": 72.21080780029297,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 61.35625076293945,
|
|
"rewards/supergames_reward/std": 72.2108154296875,
|
|
"step": 121,
|
|
"step_time": 12.588820598000893
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 297.0,
|
|
"completions/max_terminated_length": 297.0,
|
|
"completions/mean_length": 223.375,
|
|
"completions/mean_terminated_length": 223.375,
|
|
"completions/min_length": 112.0,
|
|
"completions/min_terminated_length": 112.0,
|
|
"entropy": 0.4537277817726135,
|
|
"epoch": 0.61,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.4382436275482178,
|
|
"learning_rate": 3.95e-07,
|
|
"loss": -0.1038316935300827,
|
|
"num_tokens": 1042548.0,
|
|
"reward": 5.658749580383301,
|
|
"reward_std": 83.77420043945312,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 5.558750152587891,
|
|
"rewards/supergames_reward/std": 83.77420043945312,
|
|
"step": 122,
|
|
"step_time": 11.39525846898323
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 456.0,
|
|
"completions/max_terminated_length": 456.0,
|
|
"completions/mean_length": 314.25,
|
|
"completions/mean_terminated_length": 314.25,
|
|
"completions/min_length": 95.0,
|
|
"completions/min_terminated_length": 95.0,
|
|
"entropy": 0.4113953411579132,
|
|
"epoch": 0.615,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.449777841567993,
|
|
"learning_rate": 3.8999999999999997e-07,
|
|
"loss": -0.1303664743900299,
|
|
"num_tokens": 1054422.0,
|
|
"reward": 35.98249816894531,
|
|
"reward_std": 88.6131362915039,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 35.88249969482422,
|
|
"rewards/supergames_reward/std": 88.6131362915039,
|
|
"step": 123,
|
|
"step_time": 17.001118954998674
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 242.0,
|
|
"completions/max_terminated_length": 242.0,
|
|
"completions/mean_length": 172.5,
|
|
"completions/mean_terminated_length": 172.5,
|
|
"completions/min_length": 137.0,
|
|
"completions/min_terminated_length": 137.0,
|
|
"entropy": 0.4670729339122772,
|
|
"epoch": 0.62,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.951946973800659,
|
|
"learning_rate": 3.8499999999999997e-07,
|
|
"loss": 0.05888795852661133,
|
|
"num_tokens": 1062210.0,
|
|
"reward": 43.368751525878906,
|
|
"reward_std": 40.30733871459961,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 43.26874542236328,
|
|
"rewards/supergames_reward/std": 40.307342529296875,
|
|
"step": 124,
|
|
"step_time": 9.142582255997695
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 148.0,
|
|
"completions/max_terminated_length": 148.0,
|
|
"completions/mean_length": 106.625,
|
|
"completions/mean_terminated_length": 106.625,
|
|
"completions/min_length": 79.0,
|
|
"completions/min_terminated_length": 79.0,
|
|
"entropy": 0.30219921469688416,
|
|
"epoch": 0.625,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.391322135925293,
|
|
"learning_rate": 3.7999999999999996e-07,
|
|
"loss": 0.14367049932479858,
|
|
"num_tokens": 1067599.0,
|
|
"reward": 42.63374710083008,
|
|
"reward_std": 58.20897674560547,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 42.533748626708984,
|
|
"rewards/supergames_reward/std": 58.208984375,
|
|
"step": 125,
|
|
"step_time": 5.636063549987739
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 243.0,
|
|
"completions/max_terminated_length": 243.0,
|
|
"completions/mean_length": 179.25,
|
|
"completions/mean_terminated_length": 179.25,
|
|
"completions/min_length": 134.0,
|
|
"completions/min_terminated_length": 134.0,
|
|
"entropy": 0.4834887683391571,
|
|
"epoch": 0.63,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.9391286373138428,
|
|
"learning_rate": 3.75e-07,
|
|
"loss": -0.06614185124635696,
|
|
"num_tokens": 1075457.0,
|
|
"reward": 8.302498817443848,
|
|
"reward_std": 75.10364532470703,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 8.202500343322754,
|
|
"rewards/supergames_reward/std": 75.10364532470703,
|
|
"step": 126,
|
|
"step_time": 9.130600645992672
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 196.0,
|
|
"completions/max_terminated_length": 196.0,
|
|
"completions/mean_length": 157.0,
|
|
"completions/mean_terminated_length": 157.0,
|
|
"completions/min_length": 117.0,
|
|
"completions/min_terminated_length": 117.0,
|
|
"entropy": 0.4801807701587677,
|
|
"epoch": 0.635,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.425615310668945,
|
|
"learning_rate": 3.7e-07,
|
|
"loss": -0.08830522745847702,
|
|
"num_tokens": 1083129.0,
|
|
"reward": -10.563751220703125,
|
|
"reward_std": 64.00443267822266,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -10.663749694824219,
|
|
"rewards/supergames_reward/std": 64.00444030761719,
|
|
"step": 127,
|
|
"step_time": 7.4972667430120055
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 412.0,
|
|
"completions/max_terminated_length": 412.0,
|
|
"completions/mean_length": 239.25,
|
|
"completions/mean_terminated_length": 239.25,
|
|
"completions/min_length": 89.0,
|
|
"completions/min_terminated_length": 89.0,
|
|
"entropy": 0.4667063355445862,
|
|
"epoch": 0.64,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.199113130569458,
|
|
"learning_rate": 3.65e-07,
|
|
"loss": -0.043856702744960785,
|
|
"num_tokens": 1094355.0,
|
|
"reward": -55.89250183105469,
|
|
"reward_std": 53.05980682373047,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -55.99250030517578,
|
|
"rewards/supergames_reward/std": 53.05980682373047,
|
|
"step": 128,
|
|
"step_time": 15.429580625001108
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 370.0,
|
|
"completions/max_terminated_length": 370.0,
|
|
"completions/mean_length": 244.75,
|
|
"completions/mean_terminated_length": 244.75,
|
|
"completions/min_length": 127.0,
|
|
"completions/min_terminated_length": 127.0,
|
|
"entropy": 0.57573401927948,
|
|
"epoch": 0.645,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.6800079345703125,
|
|
"learning_rate": 3.6e-07,
|
|
"loss": 0.09521033614873886,
|
|
"num_tokens": 1105633.0,
|
|
"reward": -23.35250473022461,
|
|
"reward_std": 102.31401062011719,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -23.452499389648438,
|
|
"rewards/supergames_reward/std": 102.31402587890625,
|
|
"step": 129,
|
|
"step_time": 13.920327747007832
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 258.0,
|
|
"completions/max_terminated_length": 258.0,
|
|
"completions/mean_length": 188.25,
|
|
"completions/mean_terminated_length": 188.25,
|
|
"completions/min_length": 131.0,
|
|
"completions/min_terminated_length": 131.0,
|
|
"entropy": 0.5341438055038452,
|
|
"epoch": 0.65,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.97955060005188,
|
|
"learning_rate": 3.55e-07,
|
|
"loss": -0.09512650966644287,
|
|
"num_tokens": 1113571.0,
|
|
"reward": 29.181249618530273,
|
|
"reward_std": 80.10585021972656,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 29.081249237060547,
|
|
"rewards/supergames_reward/std": 80.10585021972656,
|
|
"step": 130,
|
|
"step_time": 9.69286120700417
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 257.0,
|
|
"completions/max_terminated_length": 257.0,
|
|
"completions/mean_length": 180.75,
|
|
"completions/mean_terminated_length": 180.75,
|
|
"completions/min_length": 136.0,
|
|
"completions/min_terminated_length": 136.0,
|
|
"entropy": 0.46591487526893616,
|
|
"epoch": 0.655,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.3518130779266357,
|
|
"learning_rate": 3.5e-07,
|
|
"loss": 0.04714229330420494,
|
|
"num_tokens": 1121457.0,
|
|
"reward": -8.287501335144043,
|
|
"reward_std": 76.35875701904297,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -8.387499809265137,
|
|
"rewards/supergames_reward/std": 76.35875701904297,
|
|
"step": 131,
|
|
"step_time": 9.636637121002423
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 190.0,
|
|
"completions/max_terminated_length": 190.0,
|
|
"completions/mean_length": 147.75,
|
|
"completions/mean_terminated_length": 147.75,
|
|
"completions/min_length": 90.0,
|
|
"completions/min_terminated_length": 90.0,
|
|
"entropy": 0.4407735764980316,
|
|
"epoch": 0.66,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.498741626739502,
|
|
"learning_rate": 3.45e-07,
|
|
"loss": -0.03996007889509201,
|
|
"num_tokens": 1129095.0,
|
|
"reward": 0.1999988555908203,
|
|
"reward_std": 84.63554382324219,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 0.125,
|
|
"rewards/supergames_reward/std": 84.6017074584961,
|
|
"step": 132,
|
|
"step_time": 7.510833573003765
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 358.0,
|
|
"completions/max_terminated_length": 358.0,
|
|
"completions/mean_length": 213.875,
|
|
"completions/mean_terminated_length": 213.875,
|
|
"completions/min_length": 137.0,
|
|
"completions/min_terminated_length": 137.0,
|
|
"entropy": 0.5186149477958679,
|
|
"epoch": 0.665,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.3987107276916504,
|
|
"learning_rate": 3.4000000000000003e-07,
|
|
"loss": 0.1863594949245453,
|
|
"num_tokens": 1139110.0,
|
|
"reward": -21.328752517700195,
|
|
"reward_std": 88.9102783203125,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -21.428749084472656,
|
|
"rewards/supergames_reward/std": 88.9102783203125,
|
|
"step": 133,
|
|
"step_time": 13.393839450000087
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 230.0,
|
|
"completions/max_terminated_length": 230.0,
|
|
"completions/mean_length": 169.875,
|
|
"completions/mean_terminated_length": 169.875,
|
|
"completions/min_length": 121.0,
|
|
"completions/min_terminated_length": 121.0,
|
|
"entropy": 0.49445784091949463,
|
|
"epoch": 0.67,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.479673862457275,
|
|
"learning_rate": 3.35e-07,
|
|
"loss": 0.03618276119232178,
|
|
"num_tokens": 1146885.0,
|
|
"reward": -14.042501449584961,
|
|
"reward_std": 83.22671508789062,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -14.142499923706055,
|
|
"rewards/supergames_reward/std": 83.22671508789062,
|
|
"step": 134,
|
|
"step_time": 8.703399487014394
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 168.0,
|
|
"completions/max_terminated_length": 168.0,
|
|
"completions/mean_length": 124.75,
|
|
"completions/mean_terminated_length": 124.75,
|
|
"completions/min_length": 83.0,
|
|
"completions/min_terminated_length": 83.0,
|
|
"entropy": 0.2997814118862152,
|
|
"epoch": 0.675,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.440976619720459,
|
|
"learning_rate": 3.3e-07,
|
|
"loss": -0.029299044981598854,
|
|
"num_tokens": 1152427.0,
|
|
"reward": 45.274993896484375,
|
|
"reward_std": 44.401519775390625,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 45.17499923706055,
|
|
"rewards/supergames_reward/std": 44.40152359008789,
|
|
"step": 135,
|
|
"step_time": 6.334954546997324
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 161.0,
|
|
"completions/max_terminated_length": 161.0,
|
|
"completions/mean_length": 123.625,
|
|
"completions/mean_terminated_length": 123.625,
|
|
"completions/min_length": 93.0,
|
|
"completions/min_terminated_length": 93.0,
|
|
"entropy": 0.3462243974208832,
|
|
"epoch": 0.68,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.1878814697265625,
|
|
"learning_rate": 3.25e-07,
|
|
"loss": -0.10772529244422913,
|
|
"num_tokens": 1157952.0,
|
|
"reward": 44.186248779296875,
|
|
"reward_std": 51.78197479248047,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 44.08625030517578,
|
|
"rewards/supergames_reward/std": 51.7819709777832,
|
|
"step": 136,
|
|
"step_time": 6.067531232984038
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 155.0,
|
|
"completions/max_terminated_length": 155.0,
|
|
"completions/mean_length": 106.75,
|
|
"completions/mean_terminated_length": 106.75,
|
|
"completions/min_length": 74.0,
|
|
"completions/min_terminated_length": 74.0,
|
|
"entropy": 0.3662136197090149,
|
|
"epoch": 0.685,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.508971214294434,
|
|
"learning_rate": 3.2e-07,
|
|
"loss": -0.11172202974557877,
|
|
"num_tokens": 1163334.0,
|
|
"reward": 53.0574951171875,
|
|
"reward_std": 58.36653518676758,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 52.957496643066406,
|
|
"rewards/supergames_reward/std": 58.36653518676758,
|
|
"step": 137,
|
|
"step_time": 5.8906258280039765
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 175.0,
|
|
"completions/max_terminated_length": 175.0,
|
|
"completions/mean_length": 125.375,
|
|
"completions/mean_terminated_length": 125.375,
|
|
"completions/min_length": 90.0,
|
|
"completions/min_terminated_length": 90.0,
|
|
"entropy": 0.3352622091770172,
|
|
"epoch": 0.69,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.1937079429626465,
|
|
"learning_rate": 3.15e-07,
|
|
"loss": 0.08307601511478424,
|
|
"num_tokens": 1168929.0,
|
|
"reward": 11.796250343322754,
|
|
"reward_std": 53.48870849609375,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 11.696250915527344,
|
|
"rewards/supergames_reward/std": 53.48870849609375,
|
|
"step": 138,
|
|
"step_time": 6.595962718012743
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 254.0,
|
|
"completions/max_terminated_length": 254.0,
|
|
"completions/mean_length": 193.0,
|
|
"completions/mean_terminated_length": 193.0,
|
|
"completions/min_length": 111.0,
|
|
"completions/min_terminated_length": 111.0,
|
|
"entropy": 0.5486535429954529,
|
|
"epoch": 0.695,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.861871242523193,
|
|
"learning_rate": 3.1e-07,
|
|
"loss": 0.0321456640958786,
|
|
"num_tokens": 1176937.0,
|
|
"reward": 50.463748931884766,
|
|
"reward_std": 70.37553405761719,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 50.38875198364258,
|
|
"rewards/supergames_reward/std": 70.31440734863281,
|
|
"step": 139,
|
|
"step_time": 9.544256388006033
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 261.0,
|
|
"completions/max_terminated_length": 261.0,
|
|
"completions/mean_length": 189.5,
|
|
"completions/mean_terminated_length": 189.5,
|
|
"completions/min_length": 95.0,
|
|
"completions/min_terminated_length": 95.0,
|
|
"entropy": 0.42230603098869324,
|
|
"epoch": 0.7,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.830677032470703,
|
|
"learning_rate": 3.05e-07,
|
|
"loss": -0.07222295552492142,
|
|
"num_tokens": 1186789.0,
|
|
"reward": 33.44999694824219,
|
|
"reward_std": 67.1298599243164,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 33.349998474121094,
|
|
"rewards/supergames_reward/std": 67.1298599243164,
|
|
"step": 140,
|
|
"step_time": 10.002322309010196
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 282.0,
|
|
"completions/max_terminated_length": 282.0,
|
|
"completions/mean_length": 197.375,
|
|
"completions/mean_terminated_length": 197.375,
|
|
"completions/min_length": 92.0,
|
|
"completions/min_terminated_length": 92.0,
|
|
"entropy": 0.33588480949401855,
|
|
"epoch": 0.705,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.028353452682495,
|
|
"learning_rate": 3e-07,
|
|
"loss": -0.07543312013149261,
|
|
"num_tokens": 1196672.0,
|
|
"reward": 17.10249900817871,
|
|
"reward_std": 73.94812774658203,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 17.002500534057617,
|
|
"rewards/supergames_reward/std": 73.94813537597656,
|
|
"step": 141,
|
|
"step_time": 10.684560916008195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 235.0,
|
|
"completions/max_terminated_length": 235.0,
|
|
"completions/mean_length": 209.25,
|
|
"completions/mean_terminated_length": 209.25,
|
|
"completions/min_length": 167.0,
|
|
"completions/min_terminated_length": 167.0,
|
|
"entropy": 0.3622831702232361,
|
|
"epoch": 0.71,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.047222137451172,
|
|
"learning_rate": 2.95e-07,
|
|
"loss": -0.04569581151008606,
|
|
"num_tokens": 1206658.0,
|
|
"reward": 49.57624816894531,
|
|
"reward_std": 78.21060180664062,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 49.476253509521484,
|
|
"rewards/supergames_reward/std": 78.21060943603516,
|
|
"step": 142,
|
|
"step_time": 8.97679864402744
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 161.0,
|
|
"completions/max_terminated_length": 161.0,
|
|
"completions/mean_length": 116.25,
|
|
"completions/mean_terminated_length": 116.25,
|
|
"completions/min_length": 50.0,
|
|
"completions/min_terminated_length": 50.0,
|
|
"entropy": 0.43277010321617126,
|
|
"epoch": 0.715,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.345203876495361,
|
|
"learning_rate": 2.9e-07,
|
|
"loss": -0.16899466514587402,
|
|
"num_tokens": 1212140.0,
|
|
"reward": 27.42875099182129,
|
|
"reward_std": 48.59138870239258,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 27.328750610351562,
|
|
"rewards/supergames_reward/std": 48.591392517089844,
|
|
"step": 143,
|
|
"step_time": 6.1606669370085
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 205.0,
|
|
"completions/max_terminated_length": 205.0,
|
|
"completions/mean_length": 173.75,
|
|
"completions/mean_terminated_length": 173.75,
|
|
"completions/min_length": 134.0,
|
|
"completions/min_terminated_length": 134.0,
|
|
"entropy": 0.43806371092796326,
|
|
"epoch": 0.72,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.858840227127075,
|
|
"learning_rate": 2.8499999999999997e-07,
|
|
"loss": -0.06502802670001984,
|
|
"num_tokens": 1219954.0,
|
|
"reward": 38.69249725341797,
|
|
"reward_std": 68.78203582763672,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 38.592498779296875,
|
|
"rewards/supergames_reward/std": 68.78202819824219,
|
|
"step": 144,
|
|
"step_time": 7.88639413099736
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 169.0,
|
|
"completions/max_terminated_length": 169.0,
|
|
"completions/mean_length": 110.625,
|
|
"completions/mean_terminated_length": 110.625,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 0.3383479118347168,
|
|
"epoch": 0.725,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.225733757019043,
|
|
"learning_rate": 2.8e-07,
|
|
"loss": -0.042058318853378296,
|
|
"num_tokens": 1225399.0,
|
|
"reward": 64.85874938964844,
|
|
"reward_std": 25.170387268066406,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 64.75875091552734,
|
|
"rewards/supergames_reward/std": 25.170391082763672,
|
|
"step": 145,
|
|
"step_time": 6.432729878026294
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 147.0,
|
|
"completions/max_terminated_length": 147.0,
|
|
"completions/mean_length": 129.5,
|
|
"completions/mean_terminated_length": 129.5,
|
|
"completions/min_length": 93.0,
|
|
"completions/min_terminated_length": 93.0,
|
|
"entropy": 0.3088987171649933,
|
|
"epoch": 0.73,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0,
|
|
"learning_rate": 2.75e-07,
|
|
"loss": 0.0,
|
|
"num_tokens": 1231043.0,
|
|
"reward": 100.0999984741211,
|
|
"reward_std": 0.0,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 100.0,
|
|
"rewards/supergames_reward/std": 0.0,
|
|
"step": 146,
|
|
"step_time": 5.634907905012369
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 160.0,
|
|
"completions/max_terminated_length": 160.0,
|
|
"completions/mean_length": 120.5,
|
|
"completions/mean_terminated_length": 120.5,
|
|
"completions/min_length": 80.0,
|
|
"completions/min_terminated_length": 80.0,
|
|
"entropy": 0.37292322516441345,
|
|
"epoch": 0.735,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.174577713012695,
|
|
"learning_rate": 2.7e-07,
|
|
"loss": 0.11778222769498825,
|
|
"num_tokens": 1236607.0,
|
|
"reward": 34.82374572753906,
|
|
"reward_std": 58.513397216796875,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 34.72374725341797,
|
|
"rewards/supergames_reward/std": 58.513397216796875,
|
|
"step": 147,
|
|
"step_time": 6.078690663998714
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 172.0,
|
|
"completions/max_terminated_length": 172.0,
|
|
"completions/mean_length": 125.625,
|
|
"completions/mean_terminated_length": 125.625,
|
|
"completions/min_length": 88.0,
|
|
"completions/min_terminated_length": 88.0,
|
|
"entropy": 0.308654248714447,
|
|
"epoch": 0.74,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.8483362197875977,
|
|
"learning_rate": 2.65e-07,
|
|
"loss": -0.09181805700063705,
|
|
"num_tokens": 1242140.0,
|
|
"reward": 85.06375122070312,
|
|
"reward_std": 42.52893829345703,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 84.9637451171875,
|
|
"rewards/supergames_reward/std": 42.52893829345703,
|
|
"step": 148,
|
|
"step_time": 6.501119122985983
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 275.0,
|
|
"completions/max_terminated_length": 275.0,
|
|
"completions/mean_length": 211.125,
|
|
"completions/mean_terminated_length": 211.125,
|
|
"completions/min_length": 157.0,
|
|
"completions/min_terminated_length": 157.0,
|
|
"entropy": 0.5405827164649963,
|
|
"epoch": 0.745,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.774259567260742,
|
|
"learning_rate": 2.6e-07,
|
|
"loss": -0.03056861087679863,
|
|
"num_tokens": 1250205.0,
|
|
"reward": 84.63125610351562,
|
|
"reward_std": 34.3358154296875,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 84.53125,
|
|
"rewards/supergames_reward/std": 34.3358154296875,
|
|
"step": 149,
|
|
"step_time": 10.352729418984381
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 257.0,
|
|
"completions/max_terminated_length": 257.0,
|
|
"completions/mean_length": 218.5,
|
|
"completions/mean_terminated_length": 218.5,
|
|
"completions/min_length": 96.0,
|
|
"completions/min_terminated_length": 96.0,
|
|
"entropy": 0.3852894604206085,
|
|
"epoch": 0.75,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.6103010177612305,
|
|
"learning_rate": 2.55e-07,
|
|
"loss": 0.034789010882377625,
|
|
"num_tokens": 1260249.0,
|
|
"reward": 81.81375122070312,
|
|
"reward_std": 51.72132873535156,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 81.7137451171875,
|
|
"rewards/supergames_reward/std": 51.72132873535156,
|
|
"step": 150,
|
|
"step_time": 9.937249842012534
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 351.0,
|
|
"completions/max_terminated_length": 351.0,
|
|
"completions/mean_length": 232.625,
|
|
"completions/mean_terminated_length": 232.625,
|
|
"completions/min_length": 89.0,
|
|
"completions/min_terminated_length": 89.0,
|
|
"entropy": 0.4323120415210724,
|
|
"epoch": 0.755,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.640308380126953,
|
|
"learning_rate": 2.5e-07,
|
|
"loss": -0.11181996762752533,
|
|
"num_tokens": 1271430.0,
|
|
"reward": 56.33625030517578,
|
|
"reward_std": 78.97452545166016,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 56.23625183105469,
|
|
"rewards/supergames_reward/std": 78.97452545166016,
|
|
"step": 151,
|
|
"step_time": 13.323547195002902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 359.0,
|
|
"completions/max_terminated_length": 359.0,
|
|
"completions/mean_length": 247.5,
|
|
"completions/mean_terminated_length": 247.5,
|
|
"completions/min_length": 161.0,
|
|
"completions/min_terminated_length": 161.0,
|
|
"entropy": 0.4038960039615631,
|
|
"epoch": 0.76,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.5704824924468994,
|
|
"learning_rate": 2.45e-07,
|
|
"loss": 0.060525521636009216,
|
|
"num_tokens": 1282722.0,
|
|
"reward": -68.53875732421875,
|
|
"reward_std": 50.590599060058594,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -68.63874816894531,
|
|
"rewards/supergames_reward/std": 50.590599060058594,
|
|
"step": 152,
|
|
"step_time": 13.593127576023107
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 383.0,
|
|
"completions/max_terminated_length": 383.0,
|
|
"completions/mean_length": 246.375,
|
|
"completions/mean_terminated_length": 246.375,
|
|
"completions/min_length": 150.0,
|
|
"completions/min_terminated_length": 150.0,
|
|
"entropy": 0.4110933542251587,
|
|
"epoch": 0.765,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.233025312423706,
|
|
"learning_rate": 2.4e-07,
|
|
"loss": 0.06547188758850098,
|
|
"num_tokens": 1294061.0,
|
|
"reward": 95.91499328613281,
|
|
"reward_std": 11.836968421936035,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 95.81500244140625,
|
|
"rewards/supergames_reward/std": 11.836968421936035,
|
|
"step": 153,
|
|
"step_time": 14.499402174988063
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 338.0,
|
|
"completions/max_terminated_length": 338.0,
|
|
"completions/mean_length": 233.125,
|
|
"completions/mean_terminated_length": 233.125,
|
|
"completions/min_length": 88.0,
|
|
"completions/min_terminated_length": 88.0,
|
|
"entropy": 0.34559860825538635,
|
|
"epoch": 0.77,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.6887102127075195,
|
|
"learning_rate": 2.3499999999999997e-07,
|
|
"loss": 0.15905122458934784,
|
|
"num_tokens": 1304262.0,
|
|
"reward": 88.92375183105469,
|
|
"reward_std": 31.611207962036133,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 88.82374572753906,
|
|
"rewards/supergames_reward/std": 31.6112117767334,
|
|
"step": 154,
|
|
"step_time": 12.704337224975461
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 208.0,
|
|
"completions/max_terminated_length": 208.0,
|
|
"completions/mean_length": 178.5,
|
|
"completions/mean_terminated_length": 178.5,
|
|
"completions/min_length": 125.0,
|
|
"completions/min_terminated_length": 125.0,
|
|
"entropy": 0.4646885097026825,
|
|
"epoch": 0.775,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.161937236785889,
|
|
"learning_rate": 2.3e-07,
|
|
"loss": -0.04354584217071533,
|
|
"num_tokens": 1312098.0,
|
|
"reward": 27.80875015258789,
|
|
"reward_std": 60.81097412109375,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 27.708751678466797,
|
|
"rewards/supergames_reward/std": 60.81097412109375,
|
|
"step": 155,
|
|
"step_time": 7.911955468007363
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 391.0,
|
|
"completions/max_terminated_length": 391.0,
|
|
"completions/mean_length": 235.375,
|
|
"completions/mean_terminated_length": 235.375,
|
|
"completions/min_length": 118.0,
|
|
"completions/min_terminated_length": 118.0,
|
|
"entropy": 0.40985700488090515,
|
|
"epoch": 0.78,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.12153959274292,
|
|
"learning_rate": 2.25e-07,
|
|
"loss": -0.024320494383573532,
|
|
"num_tokens": 1322301.0,
|
|
"reward": 64.14624786376953,
|
|
"reward_std": 49.90530014038086,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 64.04624938964844,
|
|
"rewards/supergames_reward/std": 49.90530014038086,
|
|
"step": 156,
|
|
"step_time": 14.631677505996777
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 330.0,
|
|
"completions/max_terminated_length": 330.0,
|
|
"completions/mean_length": 220.625,
|
|
"completions/mean_terminated_length": 220.625,
|
|
"completions/min_length": 93.0,
|
|
"completions/min_terminated_length": 93.0,
|
|
"entropy": 0.3543764054775238,
|
|
"epoch": 0.785,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.0111091136932373,
|
|
"learning_rate": 2.1999999999999998e-07,
|
|
"loss": -0.13996481895446777,
|
|
"num_tokens": 1333386.0,
|
|
"reward": 3.6812496185302734,
|
|
"reward_std": 103.5096664428711,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 3.606250762939453,
|
|
"rewards/supergames_reward/std": 103.48104858398438,
|
|
"step": 157,
|
|
"step_time": 12.613216657977318
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 225.0,
|
|
"completions/max_terminated_length": 225.0,
|
|
"completions/mean_length": 193.125,
|
|
"completions/mean_terminated_length": 193.125,
|
|
"completions/min_length": 157.0,
|
|
"completions/min_terminated_length": 157.0,
|
|
"entropy": 0.5682947635650635,
|
|
"epoch": 0.79,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.67927885055542,
|
|
"learning_rate": 2.1499999999999998e-07,
|
|
"loss": 0.0535324327647686,
|
|
"num_tokens": 1341355.0,
|
|
"reward": -19.78999900817871,
|
|
"reward_std": 69.72611236572266,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -19.889997482299805,
|
|
"rewards/supergames_reward/std": 69.72611236572266,
|
|
"step": 158,
|
|
"step_time": 8.585773958999198
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 316.0,
|
|
"completions/max_terminated_length": 316.0,
|
|
"completions/mean_length": 237.625,
|
|
"completions/mean_terminated_length": 237.625,
|
|
"completions/min_length": 156.0,
|
|
"completions/min_terminated_length": 156.0,
|
|
"entropy": 0.3637359142303467,
|
|
"epoch": 0.795,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.033409357070923,
|
|
"learning_rate": 2.0999999999999997e-07,
|
|
"loss": 0.018884683027863503,
|
|
"num_tokens": 1352632.0,
|
|
"reward": 52.063751220703125,
|
|
"reward_std": 65.35551452636719,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 51.96375274658203,
|
|
"rewards/supergames_reward/std": 65.35551452636719,
|
|
"step": 159,
|
|
"step_time": 12.306456914986484
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 225.0,
|
|
"completions/max_terminated_length": 225.0,
|
|
"completions/mean_length": 184.0,
|
|
"completions/mean_terminated_length": 184.0,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 0.47005388140678406,
|
|
"epoch": 0.8,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.585808515548706,
|
|
"learning_rate": 2.0499999999999997e-07,
|
|
"loss": -0.021751077845692635,
|
|
"num_tokens": 1360528.0,
|
|
"reward": 33.73124694824219,
|
|
"reward_std": 78.67134857177734,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 33.631248474121094,
|
|
"rewards/supergames_reward/std": 78.67134857177734,
|
|
"step": 160,
|
|
"step_time": 8.517202848976012
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 456.0,
|
|
"completions/max_terminated_length": 456.0,
|
|
"completions/mean_length": 248.75,
|
|
"completions/mean_terminated_length": 248.75,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 0.39807969331741333,
|
|
"epoch": 0.805,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.8931398391723633,
|
|
"learning_rate": 2e-07,
|
|
"loss": -0.05018138885498047,
|
|
"num_tokens": 1371854.0,
|
|
"reward": 4.002498626708984,
|
|
"reward_std": 103.03063201904297,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 3.9274978637695312,
|
|
"rewards/supergames_reward/std": 103.00178527832031,
|
|
"step": 161,
|
|
"step_time": 17.225145012984285
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 400.0,
|
|
"completions/max_terminated_length": 400.0,
|
|
"completions/mean_length": 231.125,
|
|
"completions/mean_terminated_length": 231.125,
|
|
"completions/min_length": 95.0,
|
|
"completions/min_terminated_length": 95.0,
|
|
"entropy": 0.49391740560531616,
|
|
"epoch": 0.81,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.5471365451812744,
|
|
"learning_rate": 1.9499999999999999e-07,
|
|
"loss": -0.18011468648910522,
|
|
"num_tokens": 1383015.0,
|
|
"reward": 67.2449951171875,
|
|
"reward_std": 69.01223754882812,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 67.14500427246094,
|
|
"rewards/supergames_reward/std": 69.01224517822266,
|
|
"step": 162,
|
|
"step_time": 15.188760302000446
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 262.0,
|
|
"completions/max_terminated_length": 262.0,
|
|
"completions/mean_length": 221.875,
|
|
"completions/mean_terminated_length": 221.875,
|
|
"completions/min_length": 89.0,
|
|
"completions/min_terminated_length": 89.0,
|
|
"entropy": 0.388366162776947,
|
|
"epoch": 0.815,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.662609815597534,
|
|
"learning_rate": 1.8999999999999998e-07,
|
|
"loss": 1.8455075405654497e-05,
|
|
"num_tokens": 1393150.0,
|
|
"reward": 59.23624801635742,
|
|
"reward_std": 76.00599670410156,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 59.13624954223633,
|
|
"rewards/supergames_reward/std": 76.00599670410156,
|
|
"step": 163,
|
|
"step_time": 10.180389262997778
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 396.0,
|
|
"completions/max_terminated_length": 396.0,
|
|
"completions/mean_length": 323.625,
|
|
"completions/mean_terminated_length": 323.625,
|
|
"completions/min_length": 256.0,
|
|
"completions/min_terminated_length": 256.0,
|
|
"entropy": 0.45717528462409973,
|
|
"epoch": 0.82,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.580068826675415,
|
|
"learning_rate": 1.85e-07,
|
|
"loss": 0.024594159796833992,
|
|
"num_tokens": 1405067.0,
|
|
"reward": 6.318748474121094,
|
|
"reward_std": 88.21497344970703,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 6.21875,
|
|
"rewards/supergames_reward/std": 88.21497344970703,
|
|
"step": 164,
|
|
"step_time": 15.074436638999032
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 163.0,
|
|
"completions/max_terminated_length": 163.0,
|
|
"completions/mean_length": 133.875,
|
|
"completions/mean_terminated_length": 133.875,
|
|
"completions/min_length": 90.0,
|
|
"completions/min_terminated_length": 90.0,
|
|
"entropy": 0.3449660539627075,
|
|
"epoch": 0.825,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.045867443084717,
|
|
"learning_rate": 1.8e-07,
|
|
"loss": -0.02517889253795147,
|
|
"num_tokens": 1410730.0,
|
|
"reward": 84.35499572753906,
|
|
"reward_std": 38.135921478271484,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 84.2550048828125,
|
|
"rewards/supergames_reward/std": 38.13592529296875,
|
|
"step": 165,
|
|
"step_time": 6.275273430015659
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 144.0,
|
|
"completions/max_terminated_length": 144.0,
|
|
"completions/mean_length": 105.875,
|
|
"completions/mean_terminated_length": 105.875,
|
|
"completions/min_length": 83.0,
|
|
"completions/min_terminated_length": 83.0,
|
|
"entropy": 0.286411851644516,
|
|
"epoch": 0.83,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.855896472930908,
|
|
"learning_rate": 1.75e-07,
|
|
"loss": 0.014497723430395126,
|
|
"num_tokens": 1416081.0,
|
|
"reward": 48.40374755859375,
|
|
"reward_std": 47.61183547973633,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 48.303749084472656,
|
|
"rewards/supergames_reward/std": 47.611839294433594,
|
|
"step": 166,
|
|
"step_time": 5.5562305669882335
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.0,
|
|
"completions/max_terminated_length": 269.0,
|
|
"completions/mean_length": 226.125,
|
|
"completions/mean_terminated_length": 226.125,
|
|
"completions/min_length": 146.0,
|
|
"completions/min_terminated_length": 146.0,
|
|
"entropy": 0.39676305651664734,
|
|
"epoch": 0.835,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.633286237716675,
|
|
"learning_rate": 1.7000000000000001e-07,
|
|
"loss": -0.008210637606680393,
|
|
"num_tokens": 1426194.0,
|
|
"reward": 67.51000213623047,
|
|
"reward_std": 58.9198112487793,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 67.41000366210938,
|
|
"rewards/supergames_reward/std": 58.9198112487793,
|
|
"step": 167,
|
|
"step_time": 10.435288048989605
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 210.0,
|
|
"completions/max_terminated_length": 210.0,
|
|
"completions/mean_length": 168.5,
|
|
"completions/mean_terminated_length": 168.5,
|
|
"completions/min_length": 137.0,
|
|
"completions/min_terminated_length": 137.0,
|
|
"entropy": 0.4233987629413605,
|
|
"epoch": 0.84,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.654274940490723,
|
|
"learning_rate": 1.65e-07,
|
|
"loss": -0.0319184847176075,
|
|
"num_tokens": 1433990.0,
|
|
"reward": 50.342498779296875,
|
|
"reward_std": 67.66273498535156,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 50.26750183105469,
|
|
"rewards/supergames_reward/std": 67.5992202758789,
|
|
"step": 168,
|
|
"step_time": 8.111304188991198
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 397.0,
|
|
"completions/max_terminated_length": 397.0,
|
|
"completions/mean_length": 217.75,
|
|
"completions/mean_terminated_length": 217.75,
|
|
"completions/min_length": 90.0,
|
|
"completions/min_terminated_length": 90.0,
|
|
"entropy": 0.41511332988739014,
|
|
"epoch": 0.845,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.156782150268555,
|
|
"learning_rate": 1.6e-07,
|
|
"loss": 0.30166593194007874,
|
|
"num_tokens": 1444068.0,
|
|
"reward": 48.64500045776367,
|
|
"reward_std": 51.50307846069336,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 48.54499816894531,
|
|
"rewards/supergames_reward/std": 51.50307846069336,
|
|
"step": 169,
|
|
"step_time": 14.915170635009417
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 245.0,
|
|
"completions/max_terminated_length": 245.0,
|
|
"completions/mean_length": 218.625,
|
|
"completions/mean_terminated_length": 218.625,
|
|
"completions/min_length": 140.0,
|
|
"completions/min_terminated_length": 140.0,
|
|
"entropy": 0.3981763422489166,
|
|
"epoch": 0.85,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.818580389022827,
|
|
"learning_rate": 1.55e-07,
|
|
"loss": -0.08915036171674728,
|
|
"num_tokens": 1454153.0,
|
|
"reward": 82.69249725341797,
|
|
"reward_std": 24.65846824645996,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 82.59249877929688,
|
|
"rewards/supergames_reward/std": 24.65846824645996,
|
|
"step": 170,
|
|
"step_time": 9.446723325003404
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 386.0,
|
|
"completions/max_terminated_length": 386.0,
|
|
"completions/mean_length": 280.375,
|
|
"completions/mean_terminated_length": 280.375,
|
|
"completions/min_length": 98.0,
|
|
"completions/min_terminated_length": 98.0,
|
|
"entropy": 0.39121532440185547,
|
|
"epoch": 0.855,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.067988634109497,
|
|
"learning_rate": 1.5e-07,
|
|
"loss": -0.28749391436576843,
|
|
"num_tokens": 1465692.0,
|
|
"reward": 66.02375030517578,
|
|
"reward_std": 70.814697265625,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 65.92375183105469,
|
|
"rewards/supergames_reward/std": 70.814697265625,
|
|
"step": 171,
|
|
"step_time": 14.603269515006104
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 188.0,
|
|
"completions/max_terminated_length": 188.0,
|
|
"completions/mean_length": 131.125,
|
|
"completions/mean_terminated_length": 131.125,
|
|
"completions/min_length": 87.0,
|
|
"completions/min_terminated_length": 87.0,
|
|
"entropy": 0.435172438621521,
|
|
"epoch": 0.86,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.832774639129639,
|
|
"learning_rate": 1.45e-07,
|
|
"loss": -0.1204412579536438,
|
|
"num_tokens": 1471253.0,
|
|
"reward": 39.80875015258789,
|
|
"reward_std": 51.58554458618164,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 39.7087516784668,
|
|
"rewards/supergames_reward/std": 51.585548400878906,
|
|
"step": 172,
|
|
"step_time": 7.057335219986271
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 391.0,
|
|
"completions/max_terminated_length": 391.0,
|
|
"completions/mean_length": 271.375,
|
|
"completions/mean_terminated_length": 271.375,
|
|
"completions/min_length": 176.0,
|
|
"completions/min_terminated_length": 176.0,
|
|
"entropy": 0.4081321060657501,
|
|
"epoch": 0.865,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.2880308628082275,
|
|
"learning_rate": 1.4e-07,
|
|
"loss": 0.03748488798737526,
|
|
"num_tokens": 1482744.0,
|
|
"reward": 53.62249755859375,
|
|
"reward_std": 75.1306381225586,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 53.522499084472656,
|
|
"rewards/supergames_reward/std": 75.1306381225586,
|
|
"step": 173,
|
|
"step_time": 14.96632127999328
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 304.0,
|
|
"completions/max_terminated_length": 304.0,
|
|
"completions/mean_length": 203.875,
|
|
"completions/mean_terminated_length": 203.875,
|
|
"completions/min_length": 140.0,
|
|
"completions/min_terminated_length": 140.0,
|
|
"entropy": 0.4152996242046356,
|
|
"epoch": 0.87,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.1565890312194824,
|
|
"learning_rate": 1.35e-07,
|
|
"loss": -0.047037553042173386,
|
|
"num_tokens": 1492679.0,
|
|
"reward": 37.48374938964844,
|
|
"reward_std": 72.71965789794922,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 37.383750915527344,
|
|
"rewards/supergames_reward/std": 72.71966552734375,
|
|
"step": 174,
|
|
"step_time": 11.566132662002929
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 187.0,
|
|
"completions/max_terminated_length": 187.0,
|
|
"completions/mean_length": 133.375,
|
|
"completions/mean_terminated_length": 133.375,
|
|
"completions/min_length": 95.0,
|
|
"completions/min_terminated_length": 95.0,
|
|
"entropy": 0.3746980130672455,
|
|
"epoch": 0.875,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.724977970123291,
|
|
"learning_rate": 1.3e-07,
|
|
"loss": 0.05197479575872421,
|
|
"num_tokens": 1498330.0,
|
|
"reward": 37.86625289916992,
|
|
"reward_std": 52.43158721923828,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 37.76625061035156,
|
|
"rewards/supergames_reward/std": 52.43158721923828,
|
|
"step": 175,
|
|
"step_time": 7.080777855007909
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 448.0,
|
|
"completions/max_terminated_length": 448.0,
|
|
"completions/mean_length": 277.25,
|
|
"completions/mean_terminated_length": 277.25,
|
|
"completions/min_length": 96.0,
|
|
"completions/min_terminated_length": 96.0,
|
|
"entropy": 0.397225022315979,
|
|
"epoch": 0.88,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.438502788543701,
|
|
"learning_rate": 1.25e-07,
|
|
"loss": 0.1022070124745369,
|
|
"num_tokens": 1509860.0,
|
|
"reward": -41.323753356933594,
|
|
"reward_std": 79.704345703125,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -41.42374801635742,
|
|
"rewards/supergames_reward/std": 79.704345703125,
|
|
"step": 176,
|
|
"step_time": 16.852883081999607
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 187.0,
|
|
"completions/max_terminated_length": 187.0,
|
|
"completions/mean_length": 145.875,
|
|
"completions/mean_terminated_length": 145.875,
|
|
"completions/min_length": 92.0,
|
|
"completions/min_terminated_length": 92.0,
|
|
"entropy": 0.34882763028144836,
|
|
"epoch": 0.885,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.683617115020752,
|
|
"learning_rate": 1.2e-07,
|
|
"loss": 0.0609549917280674,
|
|
"num_tokens": 1515587.0,
|
|
"reward": 84.27999877929688,
|
|
"reward_std": 38.06671905517578,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 84.17999267578125,
|
|
"rewards/supergames_reward/std": 38.06672286987305,
|
|
"step": 177,
|
|
"step_time": 7.09286071601673
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 217.0,
|
|
"completions/max_terminated_length": 217.0,
|
|
"completions/mean_length": 181.875,
|
|
"completions/mean_terminated_length": 181.875,
|
|
"completions/min_length": 146.0,
|
|
"completions/min_terminated_length": 146.0,
|
|
"entropy": 0.423494815826416,
|
|
"epoch": 0.89,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.3993117809295654,
|
|
"learning_rate": 1.15e-07,
|
|
"loss": 0.04823008552193642,
|
|
"num_tokens": 1523490.0,
|
|
"reward": 22.46124839782715,
|
|
"reward_std": 89.67146301269531,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 22.361249923706055,
|
|
"rewards/supergames_reward/std": 89.67146301269531,
|
|
"step": 178,
|
|
"step_time": 8.281584940006724
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.0,
|
|
"completions/max_terminated_length": 315.0,
|
|
"completions/mean_length": 273.25,
|
|
"completions/mean_terminated_length": 273.25,
|
|
"completions/min_length": 234.0,
|
|
"completions/min_terminated_length": 234.0,
|
|
"entropy": 0.3831624984741211,
|
|
"epoch": 0.895,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.7383997440338135,
|
|
"learning_rate": 1.0999999999999999e-07,
|
|
"loss": -0.010000350885093212,
|
|
"num_tokens": 1535012.0,
|
|
"reward": 2.237499237060547,
|
|
"reward_std": 93.32353210449219,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 2.137500762939453,
|
|
"rewards/supergames_reward/std": 93.32353973388672,
|
|
"step": 179,
|
|
"step_time": 12.098909514024854
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 395.0,
|
|
"completions/max_terminated_length": 395.0,
|
|
"completions/mean_length": 197.375,
|
|
"completions/mean_terminated_length": 197.375,
|
|
"completions/min_length": 132.0,
|
|
"completions/min_terminated_length": 132.0,
|
|
"entropy": 0.5201653242111206,
|
|
"epoch": 0.9,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.4596073627471924,
|
|
"learning_rate": 1.0499999999999999e-07,
|
|
"loss": -0.11710439622402191,
|
|
"num_tokens": 1543023.0,
|
|
"reward": 88.1875,
|
|
"reward_std": 33.69363784790039,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 88.0875015258789,
|
|
"rewards/supergames_reward/std": 33.693641662597656,
|
|
"step": 180,
|
|
"step_time": 14.662479719001567
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 200,
|
|
"num_input_tokens_seen": 1543023,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 20,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|