1195 lines
43 KiB
JSON
1195 lines
43 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.2,
|
|
"eval_steps": 500,
|
|
"global_step": 40,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 478.0,
|
|
"completions/max_terminated_length": 478.0,
|
|
"completions/mean_length": 191.0,
|
|
"completions/mean_terminated_length": 191.0,
|
|
"completions/min_length": 66.0,
|
|
"completions/min_terminated_length": 66.0,
|
|
"entropy": 0.5566893219947815,
|
|
"epoch": 0.005,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.370361328125,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.24658073484897614,
|
|
"num_tokens": 10832.0,
|
|
"reward": -3.3000030517578125,
|
|
"reward_std": 85.62333679199219,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -3.4000015258789062,
|
|
"rewards/supergames_reward/std": 85.62333679199219,
|
|
"step": 1,
|
|
"step_time": 22.779711072013015
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 216.0,
|
|
"completions/max_terminated_length": 216.0,
|
|
"completions/mean_length": 131.5,
|
|
"completions/mean_terminated_length": 131.5,
|
|
"completions/min_length": 60.0,
|
|
"completions/min_terminated_length": 60.0,
|
|
"entropy": 0.6395841240882874,
|
|
"epoch": 0.01,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 8.09774398803711,
|
|
"learning_rate": 9.95e-07,
|
|
"loss": 0.20567649602890015,
|
|
"num_tokens": 16404.0,
|
|
"reward": -12.422499656677246,
|
|
"reward_std": 7.134707450866699,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -12.522500038146973,
|
|
"rewards/supergames_reward/std": 7.134707927703857,
|
|
"step": 2,
|
|
"step_time": 9.960156448010821
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 287.0,
|
|
"completions/max_terminated_length": 287.0,
|
|
"completions/mean_length": 156.625,
|
|
"completions/mean_terminated_length": 156.625,
|
|
"completions/min_length": 77.0,
|
|
"completions/min_terminated_length": 77.0,
|
|
"entropy": 0.562222421169281,
|
|
"epoch": 0.015,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.9053616523742676,
|
|
"learning_rate": 9.9e-07,
|
|
"loss": 0.09602774679660797,
|
|
"num_tokens": 26953.0,
|
|
"reward": -74.94999694824219,
|
|
"reward_std": 70.73094177246094,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -75.0,
|
|
"rewards/supergames_reward/std": 70.71067810058594,
|
|
"step": 3,
|
|
"step_time": 13.686320498993155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 268.0,
|
|
"completions/max_terminated_length": 268.0,
|
|
"completions/mean_length": 154.125,
|
|
"completions/mean_terminated_length": 154.125,
|
|
"completions/min_length": 53.0,
|
|
"completions/min_terminated_length": 53.0,
|
|
"entropy": 0.7323317527770996,
|
|
"epoch": 0.02,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.933310031890869,
|
|
"learning_rate": 9.849999999999999e-07,
|
|
"loss": 0.28249427676200867,
|
|
"num_tokens": 36514.0,
|
|
"reward": -40.7599983215332,
|
|
"reward_std": 81.73140716552734,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -40.80999755859375,
|
|
"rewards/supergames_reward/std": 81.68995666503906,
|
|
"step": 4,
|
|
"step_time": 12.805880262021674
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 401.0,
|
|
"completions/max_terminated_length": 401.0,
|
|
"completions/mean_length": 205.25,
|
|
"completions/mean_terminated_length": 205.25,
|
|
"completions/min_length": 115.0,
|
|
"completions/min_terminated_length": 115.0,
|
|
"entropy": 0.6167430281639099,
|
|
"epoch": 0.025,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.344135046005249,
|
|
"learning_rate": 9.8e-07,
|
|
"loss": 0.1433994024991989,
|
|
"num_tokens": 47476.0,
|
|
"reward": -97.6612548828125,
|
|
"reward_std": 6.3321428298950195,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -97.76124572753906,
|
|
"rewards/supergames_reward/std": 6.3321428298950195,
|
|
"step": 5,
|
|
"step_time": 18.789364666008623
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 276.0,
|
|
"completions/max_terminated_length": 276.0,
|
|
"completions/mean_length": 167.25,
|
|
"completions/mean_terminated_length": 167.25,
|
|
"completions/min_length": 73.0,
|
|
"completions/min_terminated_length": 73.0,
|
|
"entropy": 0.7583790421485901,
|
|
"epoch": 0.03,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.172371864318848,
|
|
"learning_rate": 9.75e-07,
|
|
"loss": -0.12127404659986496,
|
|
"num_tokens": 55246.0,
|
|
"reward": -58.687503814697266,
|
|
"reward_std": 58.64059829711914,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -58.76250457763672,
|
|
"rewards/supergames_reward/std": 58.620460510253906,
|
|
"step": 6,
|
|
"step_time": 12.948570273991209
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 261.0,
|
|
"completions/max_terminated_length": 261.0,
|
|
"completions/mean_length": 133.875,
|
|
"completions/mean_terminated_length": 133.875,
|
|
"completions/min_length": 57.0,
|
|
"completions/min_terminated_length": 57.0,
|
|
"entropy": 0.7447654604911804,
|
|
"epoch": 0.035,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.428595066070557,
|
|
"learning_rate": 9.7e-07,
|
|
"loss": 0.10135584324598312,
|
|
"num_tokens": 60837.0,
|
|
"reward": -32.45000076293945,
|
|
"reward_std": 41.754486083984375,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -32.5,
|
|
"rewards/supergames_reward/std": 41.66190719604492,
|
|
"step": 7,
|
|
"step_time": 12.058315072004916
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 414.0,
|
|
"completions/max_terminated_length": 414.0,
|
|
"completions/mean_length": 209.125,
|
|
"completions/mean_terminated_length": 209.125,
|
|
"completions/min_length": 104.0,
|
|
"completions/min_terminated_length": 104.0,
|
|
"entropy": 0.3860666751861572,
|
|
"epoch": 0.04,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.806255578994751,
|
|
"learning_rate": 9.649999999999999e-07,
|
|
"loss": -0.33238139748573303,
|
|
"num_tokens": 70838.0,
|
|
"reward": -80.50375366210938,
|
|
"reward_std": 38.82380294799805,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -80.60375213623047,
|
|
"rewards/supergames_reward/std": 38.82379913330078,
|
|
"step": 8,
|
|
"step_time": 19.323370319994865
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 199.0,
|
|
"completions/mean_length": 247.125,
|
|
"completions/mean_terminated_length": 158.83334350585938,
|
|
"completions/min_length": 106.0,
|
|
"completions/min_terminated_length": 106.0,
|
|
"entropy": 0.5816237330436707,
|
|
"epoch": 0.045,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.3209943771362305,
|
|
"learning_rate": 9.6e-07,
|
|
"loss": 0.36752766370773315,
|
|
"num_tokens": 82135.0,
|
|
"reward": -27.688751220703125,
|
|
"reward_std": 82.8358154296875,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -27.738750457763672,
|
|
"rewards/supergames_reward/std": 82.78590393066406,
|
|
"step": 9,
|
|
"step_time": 21.948575104994234
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 240.0,
|
|
"completions/max_terminated_length": 240.0,
|
|
"completions/mean_length": 153.0,
|
|
"completions/mean_terminated_length": 153.0,
|
|
"completions/min_length": 51.0,
|
|
"completions/min_terminated_length": 51.0,
|
|
"entropy": 0.6301521062850952,
|
|
"epoch": 0.05,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.125359058380127,
|
|
"learning_rate": 9.55e-07,
|
|
"loss": 0.06129350885748863,
|
|
"num_tokens": 89807.0,
|
|
"reward": -78.39624786376953,
|
|
"reward_std": 40.17042922973633,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -78.44625091552734,
|
|
"rewards/supergames_reward/std": 40.1396484375,
|
|
"step": 10,
|
|
"step_time": 9.098202993016457
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 133.0,
|
|
"completions/max_terminated_length": 133.0,
|
|
"completions/mean_length": 112.0,
|
|
"completions/mean_terminated_length": 112.0,
|
|
"completions/min_length": 68.0,
|
|
"completions/min_terminated_length": 68.0,
|
|
"entropy": 0.5521990656852722,
|
|
"epoch": 0.055,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.616060256958008,
|
|
"learning_rate": 9.499999999999999e-07,
|
|
"loss": -0.006313305348157883,
|
|
"num_tokens": 95231.0,
|
|
"reward": -12.161249160766602,
|
|
"reward_std": 6.39578104019165,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -12.261249542236328,
|
|
"rewards/supergames_reward/std": 6.39578104019165,
|
|
"step": 11,
|
|
"step_time": 5.136311663984088
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 396.0,
|
|
"completions/max_terminated_length": 396.0,
|
|
"completions/mean_length": 214.875,
|
|
"completions/mean_terminated_length": 214.875,
|
|
"completions/min_length": 55.0,
|
|
"completions/min_terminated_length": 55.0,
|
|
"entropy": 0.5505905151367188,
|
|
"epoch": 0.06,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.289583206176758,
|
|
"learning_rate": 9.45e-07,
|
|
"loss": -0.29802003502845764,
|
|
"num_tokens": 106238.0,
|
|
"reward": -91.66999816894531,
|
|
"reward_std": 23.2779541015625,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -91.77000427246094,
|
|
"rewards/supergames_reward/std": 23.277956008911133,
|
|
"step": 12,
|
|
"step_time": 14.945365622988902
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 269.0,
|
|
"completions/max_terminated_length": 269.0,
|
|
"completions/mean_length": 167.375,
|
|
"completions/mean_terminated_length": 167.375,
|
|
"completions/min_length": 113.0,
|
|
"completions/min_terminated_length": 113.0,
|
|
"entropy": 0.540415346622467,
|
|
"epoch": 0.065,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.858291149139404,
|
|
"learning_rate": 9.399999999999999e-07,
|
|
"loss": 0.24004128575325012,
|
|
"num_tokens": 116937.0,
|
|
"reward": -20.38249969482422,
|
|
"reward_std": 87.55204010009766,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -20.457500457763672,
|
|
"rewards/supergames_reward/std": 87.52605438232422,
|
|
"step": 13,
|
|
"step_time": 10.482285185018554
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 254.0,
|
|
"completions/max_terminated_length": 254.0,
|
|
"completions/mean_length": 156.875,
|
|
"completions/mean_terminated_length": 156.875,
|
|
"completions/min_length": 114.0,
|
|
"completions/min_terminated_length": 114.0,
|
|
"entropy": 0.7237679958343506,
|
|
"epoch": 0.07,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.083980560302734,
|
|
"learning_rate": 9.35e-07,
|
|
"loss": -0.19025824964046478,
|
|
"num_tokens": 124640.0,
|
|
"reward": -51.397499084472656,
|
|
"reward_std": 54.486454010009766,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -51.49749755859375,
|
|
"rewards/supergames_reward/std": 54.486454010009766,
|
|
"step": 14,
|
|
"step_time": 9.597231683001155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 236.0,
|
|
"completions/max_terminated_length": 236.0,
|
|
"completions/mean_length": 141.375,
|
|
"completions/mean_terminated_length": 141.375,
|
|
"completions/min_length": 91.0,
|
|
"completions/min_terminated_length": 91.0,
|
|
"entropy": 0.754334568977356,
|
|
"epoch": 0.075,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.720428943634033,
|
|
"learning_rate": 9.3e-07,
|
|
"loss": -0.16830675303936005,
|
|
"num_tokens": 132219.0,
|
|
"reward": -56.226253509521484,
|
|
"reward_std": 60.115657806396484,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -56.32625198364258,
|
|
"rewards/supergames_reward/std": 60.11566162109375,
|
|
"step": 15,
|
|
"step_time": 8.998362872982398
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 209.0,
|
|
"completions/max_terminated_length": 209.0,
|
|
"completions/mean_length": 136.375,
|
|
"completions/mean_terminated_length": 136.375,
|
|
"completions/min_length": 97.0,
|
|
"completions/min_terminated_length": 97.0,
|
|
"entropy": 0.6280568838119507,
|
|
"epoch": 0.08,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.9681806564331055,
|
|
"learning_rate": 9.25e-07,
|
|
"loss": -0.01910916529595852,
|
|
"num_tokens": 139726.0,
|
|
"reward": -68.24000549316406,
|
|
"reward_std": 52.238914489746094,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -68.33999633789062,
|
|
"rewards/supergames_reward/std": 52.238914489746094,
|
|
"step": 16,
|
|
"step_time": 8.034480294008972
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 365.0,
|
|
"completions/max_terminated_length": 365.0,
|
|
"completions/mean_length": 211.875,
|
|
"completions/mean_terminated_length": 211.875,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"entropy": 0.899176836013794,
|
|
"epoch": 0.085,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.691746711730957,
|
|
"learning_rate": 9.2e-07,
|
|
"loss": 0.25513291358947754,
|
|
"num_tokens": 147869.0,
|
|
"reward": -99.92500305175781,
|
|
"reward_std": 0.07070883363485336,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -100.0,
|
|
"rewards/supergames_reward/std": 0.0,
|
|
"step": 17,
|
|
"step_time": 13.655792869016295
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 387.0,
|
|
"completions/max_terminated_length": 387.0,
|
|
"completions/mean_length": 238.0,
|
|
"completions/mean_terminated_length": 238.0,
|
|
"completions/min_length": 87.0,
|
|
"completions/min_terminated_length": 87.0,
|
|
"entropy": 0.5596873760223389,
|
|
"epoch": 0.09,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.6711347103118896,
|
|
"learning_rate": 9.15e-07,
|
|
"loss": -0.2628335654735565,
|
|
"num_tokens": 159101.0,
|
|
"reward": -24.900001525878906,
|
|
"reward_std": 103.50983428955078,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -25.0,
|
|
"rewards/supergames_reward/std": 103.50983428955078,
|
|
"step": 18,
|
|
"step_time": 14.784329420013819
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 277.0,
|
|
"completions/mean_length": 219.75,
|
|
"completions/mean_terminated_length": 178.00001525878906,
|
|
"completions/min_length": 90.0,
|
|
"completions/min_terminated_length": 90.0,
|
|
"entropy": 0.9245517253875732,
|
|
"epoch": 0.095,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.55502986907959,
|
|
"learning_rate": 9.1e-07,
|
|
"loss": 0.13060179352760315,
|
|
"num_tokens": 167299.0,
|
|
"reward": -86.89125061035156,
|
|
"reward_std": 36.95603561401367,
|
|
"rewards/json_format_reward/mean": 0.05000000074505806,
|
|
"rewards/json_format_reward/std": 0.09258200973272324,
|
|
"rewards/supergames_reward/mean": -86.94125366210938,
|
|
"rewards/supergames_reward/std": 36.93572235107422,
|
|
"step": 19,
|
|
"step_time": 18.832684700988466
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 267.0,
|
|
"completions/max_terminated_length": 267.0,
|
|
"completions/mean_length": 177.25,
|
|
"completions/mean_terminated_length": 177.25,
|
|
"completions/min_length": 93.0,
|
|
"completions/min_terminated_length": 93.0,
|
|
"entropy": 0.7331254482269287,
|
|
"epoch": 0.1,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.538497447967529,
|
|
"learning_rate": 9.05e-07,
|
|
"loss": -0.16418980062007904,
|
|
"num_tokens": 175141.0,
|
|
"reward": -68.52874755859375,
|
|
"reward_std": 58.48719787597656,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -68.62875366210938,
|
|
"rewards/supergames_reward/std": 58.48720169067383,
|
|
"step": 20,
|
|
"step_time": 10.090975169994636
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 300.0,
|
|
"completions/max_terminated_length": 300.0,
|
|
"completions/mean_length": 190.625,
|
|
"completions/mean_terminated_length": 190.625,
|
|
"completions/min_length": 103.0,
|
|
"completions/min_terminated_length": 103.0,
|
|
"entropy": 0.6659425497055054,
|
|
"epoch": 0.105,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.485373497009277,
|
|
"learning_rate": 9e-07,
|
|
"loss": -0.10106432437896729,
|
|
"num_tokens": 183098.0,
|
|
"reward": -71.48875427246094,
|
|
"reward_std": 55.33565139770508,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -71.5887451171875,
|
|
"rewards/supergames_reward/std": 55.33565139770508,
|
|
"step": 21,
|
|
"step_time": 11.255001295008697
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 279.0,
|
|
"completions/max_terminated_length": 279.0,
|
|
"completions/mean_length": 179.0,
|
|
"completions/mean_terminated_length": 179.0,
|
|
"completions/min_length": 103.0,
|
|
"completions/min_terminated_length": 103.0,
|
|
"entropy": 0.49365949630737305,
|
|
"epoch": 0.11,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.829419136047363,
|
|
"learning_rate": 8.95e-07,
|
|
"loss": 0.10587231814861298,
|
|
"num_tokens": 192826.0,
|
|
"reward": -75.62000274658203,
|
|
"reward_std": 45.01959991455078,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -75.69499969482422,
|
|
"rewards/supergames_reward/std": 45.00411605834961,
|
|
"step": 22,
|
|
"step_time": 10.611246599000879
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 315.0,
|
|
"completions/max_terminated_length": 315.0,
|
|
"completions/mean_length": 212.625,
|
|
"completions/mean_terminated_length": 212.625,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"entropy": 0.43159469962120056,
|
|
"epoch": 0.115,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.541125774383545,
|
|
"learning_rate": 8.9e-07,
|
|
"loss": -0.1681259125471115,
|
|
"num_tokens": 202815.0,
|
|
"reward": 23.67624855041504,
|
|
"reward_std": 67.74340057373047,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 23.576250076293945,
|
|
"rewards/supergames_reward/std": 67.743408203125,
|
|
"step": 23,
|
|
"step_time": 11.731771531980485
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 168.0,
|
|
"completions/max_terminated_length": 168.0,
|
|
"completions/mean_length": 101.625,
|
|
"completions/mean_terminated_length": 101.625,
|
|
"completions/min_length": 65.0,
|
|
"completions/min_terminated_length": 65.0,
|
|
"entropy": 0.3972298502922058,
|
|
"epoch": 0.12,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.8285417556762695,
|
|
"learning_rate": 8.85e-07,
|
|
"loss": -0.011585958302021027,
|
|
"num_tokens": 208188.0,
|
|
"reward": 58.849998474121094,
|
|
"reward_std": 56.9304084777832,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 58.75,
|
|
"rewards/supergames_reward/std": 56.9304084777832,
|
|
"step": 24,
|
|
"step_time": 6.187504950998118
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 303.0,
|
|
"completions/max_terminated_length": 303.0,
|
|
"completions/mean_length": 165.375,
|
|
"completions/mean_terminated_length": 165.375,
|
|
"completions/min_length": 69.0,
|
|
"completions/min_terminated_length": 69.0,
|
|
"entropy": 0.6339899301528931,
|
|
"epoch": 0.125,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.823799133300781,
|
|
"learning_rate": 8.799999999999999e-07,
|
|
"loss": 0.05159185826778412,
|
|
"num_tokens": 215935.0,
|
|
"reward": -23.521251678466797,
|
|
"reward_std": 74.23067474365234,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -23.621248245239258,
|
|
"rewards/supergames_reward/std": 74.23067474365234,
|
|
"step": 25,
|
|
"step_time": 11.059416300005978
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 349.0,
|
|
"completions/max_terminated_length": 349.0,
|
|
"completions/mean_length": 204.0,
|
|
"completions/mean_terminated_length": 204.0,
|
|
"completions/min_length": 143.0,
|
|
"completions/min_terminated_length": 143.0,
|
|
"entropy": 0.6462895274162292,
|
|
"epoch": 0.13,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.969292640686035,
|
|
"learning_rate": 8.75e-07,
|
|
"loss": 0.0115677984431386,
|
|
"num_tokens": 223999.0,
|
|
"reward": -74.8175048828125,
|
|
"reward_std": 40.94221115112305,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -74.89250183105469,
|
|
"rewards/supergames_reward/std": 40.924625396728516,
|
|
"step": 26,
|
|
"step_time": 12.666237785975682
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 299.0,
|
|
"completions/max_terminated_length": 299.0,
|
|
"completions/mean_length": 166.5,
|
|
"completions/mean_terminated_length": 166.5,
|
|
"completions/min_length": 87.0,
|
|
"completions/min_terminated_length": 87.0,
|
|
"entropy": 0.4756850600242615,
|
|
"epoch": 0.135,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.915069103240967,
|
|
"learning_rate": 8.699999999999999e-07,
|
|
"loss": -0.13321346044540405,
|
|
"num_tokens": 233643.0,
|
|
"reward": -52.125,
|
|
"reward_std": 19.405282974243164,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -52.20000076293945,
|
|
"rewards/supergames_reward/std": 19.33464813232422,
|
|
"step": 27,
|
|
"step_time": 11.172539752995363
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 184.0,
|
|
"completions/max_terminated_length": 184.0,
|
|
"completions/mean_length": 137.125,
|
|
"completions/mean_terminated_length": 137.125,
|
|
"completions/min_length": 100.0,
|
|
"completions/min_terminated_length": 100.0,
|
|
"entropy": 0.5834778547286987,
|
|
"epoch": 0.14,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.146666526794434,
|
|
"learning_rate": 8.65e-07,
|
|
"loss": 0.060454584658145905,
|
|
"num_tokens": 241204.0,
|
|
"reward": -68.99250793457031,
|
|
"reward_std": 51.010581970214844,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -69.09249877929688,
|
|
"rewards/supergames_reward/std": 51.010581970214844,
|
|
"step": 28,
|
|
"step_time": 7.025046669004951
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 331.0,
|
|
"completions/max_terminated_length": 331.0,
|
|
"completions/mean_length": 182.375,
|
|
"completions/mean_terminated_length": 182.375,
|
|
"completions/min_length": 106.0,
|
|
"completions/min_terminated_length": 106.0,
|
|
"entropy": 0.6368551254272461,
|
|
"epoch": 0.145,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.15535306930542,
|
|
"learning_rate": 8.599999999999999e-07,
|
|
"loss": -0.3323673903942108,
|
|
"num_tokens": 249087.0,
|
|
"reward": -44.17000198364258,
|
|
"reward_std": 80.5373306274414,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -44.27000045776367,
|
|
"rewards/supergames_reward/std": 80.5373306274414,
|
|
"step": 29,
|
|
"step_time": 12.079259724996518
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 190.0,
|
|
"completions/max_terminated_length": 190.0,
|
|
"completions/mean_length": 127.75,
|
|
"completions/mean_terminated_length": 127.75,
|
|
"completions/min_length": 80.0,
|
|
"completions/min_terminated_length": 80.0,
|
|
"entropy": 0.4491196870803833,
|
|
"epoch": 0.15,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.896186828613281,
|
|
"learning_rate": 8.55e-07,
|
|
"loss": -0.12887656688690186,
|
|
"num_tokens": 259445.0,
|
|
"reward": 57.446250915527344,
|
|
"reward_std": 38.78837585449219,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 57.34625244140625,
|
|
"rewards/supergames_reward/std": 38.78837966918945,
|
|
"step": 30,
|
|
"step_time": 7.483972909016302
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 166.0,
|
|
"completions/max_terminated_length": 166.0,
|
|
"completions/mean_length": 112.875,
|
|
"completions/mean_terminated_length": 112.875,
|
|
"completions/min_length": 85.0,
|
|
"completions/min_terminated_length": 85.0,
|
|
"entropy": 0.4858454465866089,
|
|
"epoch": 0.155,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.742906093597412,
|
|
"learning_rate": 8.499999999999999e-07,
|
|
"loss": 0.06930096447467804,
|
|
"num_tokens": 264924.0,
|
|
"reward": -21.174999237060547,
|
|
"reward_std": 31.89051628112793,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -21.25,
|
|
"rewards/supergames_reward/std": 31.819807052612305,
|
|
"step": 31,
|
|
"step_time": 6.1866529019898735
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 148.0,
|
|
"completions/max_terminated_length": 148.0,
|
|
"completions/mean_length": 118.125,
|
|
"completions/mean_terminated_length": 118.125,
|
|
"completions/min_length": 98.0,
|
|
"completions/min_terminated_length": 98.0,
|
|
"entropy": 0.45428749918937683,
|
|
"epoch": 0.16,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.663414478302002,
|
|
"learning_rate": 8.45e-07,
|
|
"loss": 0.05735419690608978,
|
|
"num_tokens": 270381.0,
|
|
"reward": 0.4687504768371582,
|
|
"reward_std": 38.66371154785156,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": 0.36875057220458984,
|
|
"rewards/supergames_reward/std": 38.6637077331543,
|
|
"step": 32,
|
|
"step_time": 5.577042757999152
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 345.0,
|
|
"completions/max_terminated_length": 345.0,
|
|
"completions/mean_length": 166.875,
|
|
"completions/mean_terminated_length": 166.875,
|
|
"completions/min_length": 86.0,
|
|
"completions/min_terminated_length": 86.0,
|
|
"entropy": 0.6107293367385864,
|
|
"epoch": 0.165,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.75006628036499,
|
|
"learning_rate": 8.399999999999999e-07,
|
|
"loss": -0.14760127663612366,
|
|
"num_tokens": 278156.0,
|
|
"reward": -87.84750366210938,
|
|
"reward_std": 26.84708023071289,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -87.94749450683594,
|
|
"rewards/supergames_reward/std": 26.847076416015625,
|
|
"step": 33,
|
|
"step_time": 12.552876825997373
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.125,
|
|
"completions/max_length": 512.0,
|
|
"completions/max_terminated_length": 360.0,
|
|
"completions/mean_length": 231.0,
|
|
"completions/mean_terminated_length": 190.85714721679688,
|
|
"completions/min_length": 74.0,
|
|
"completions/min_terminated_length": 74.0,
|
|
"entropy": 0.6089653372764587,
|
|
"epoch": 0.17,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.781267166137695,
|
|
"learning_rate": 8.349999999999999e-07,
|
|
"loss": 0.4294504225254059,
|
|
"num_tokens": 289364.0,
|
|
"reward": -99.92500305175781,
|
|
"reward_std": 0.07070883363485336,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -100.0,
|
|
"rewards/supergames_reward/std": 0.0,
|
|
"step": 34,
|
|
"step_time": 18.961819477990502
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 218.0,
|
|
"completions/max_terminated_length": 218.0,
|
|
"completions/mean_length": 134.125,
|
|
"completions/mean_terminated_length": 134.125,
|
|
"completions/min_length": 104.0,
|
|
"completions/min_terminated_length": 104.0,
|
|
"entropy": 0.46158918738365173,
|
|
"epoch": 0.175,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.016353607177734,
|
|
"learning_rate": 8.299999999999999e-07,
|
|
"loss": 0.09251818805932999,
|
|
"num_tokens": 298733.0,
|
|
"reward": -35.558753967285156,
|
|
"reward_std": 15.881204605102539,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -35.65875244140625,
|
|
"rewards/supergames_reward/std": 15.881203651428223,
|
|
"step": 35,
|
|
"step_time": 8.38491243700264
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 207.0,
|
|
"completions/max_terminated_length": 207.0,
|
|
"completions/mean_length": 156.875,
|
|
"completions/mean_terminated_length": 156.875,
|
|
"completions/min_length": 111.0,
|
|
"completions/min_terminated_length": 111.0,
|
|
"entropy": 0.5104788541793823,
|
|
"epoch": 0.18,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.592893123626709,
|
|
"learning_rate": 8.249999999999999e-07,
|
|
"loss": -0.0410832017660141,
|
|
"num_tokens": 306420.0,
|
|
"reward": 5.4212493896484375,
|
|
"reward_std": 93.50404357910156,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": 5.346250534057617,
|
|
"rewards/supergames_reward/std": 93.4718246459961,
|
|
"step": 36,
|
|
"step_time": 7.811868985998444
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 241.0,
|
|
"completions/max_terminated_length": 241.0,
|
|
"completions/mean_length": 178.0,
|
|
"completions/mean_terminated_length": 178.0,
|
|
"completions/min_length": 106.0,
|
|
"completions/min_terminated_length": 106.0,
|
|
"entropy": 0.7527878880500793,
|
|
"epoch": 0.185,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.8812546730041504,
|
|
"learning_rate": 8.199999999999999e-07,
|
|
"loss": 0.12539231777191162,
|
|
"num_tokens": 314252.0,
|
|
"reward": -89.58500671386719,
|
|
"reward_std": 29.256120681762695,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -89.66000366210938,
|
|
"rewards/supergames_reward/std": 29.24593734741211,
|
|
"step": 37,
|
|
"step_time": 8.959742914012168
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 199.0,
|
|
"completions/max_terminated_length": 199.0,
|
|
"completions/mean_length": 133.375,
|
|
"completions/mean_terminated_length": 133.375,
|
|
"completions/min_length": 87.0,
|
|
"completions/min_terminated_length": 87.0,
|
|
"entropy": 0.3347901403903961,
|
|
"epoch": 0.19,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.205414295196533,
|
|
"learning_rate": 8.149999999999999e-07,
|
|
"loss": -0.06991486996412277,
|
|
"num_tokens": 319839.0,
|
|
"reward": -14.498749732971191,
|
|
"reward_std": 13.007229804992676,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -14.598750114440918,
|
|
"rewards/supergames_reward/std": 13.007229804992676,
|
|
"step": 38,
|
|
"step_time": 7.377183554985095
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 181.0,
|
|
"completions/max_terminated_length": 181.0,
|
|
"completions/mean_length": 114.375,
|
|
"completions/mean_terminated_length": 114.375,
|
|
"completions/min_length": 58.0,
|
|
"completions/min_terminated_length": 58.0,
|
|
"entropy": 0.4445965886116028,
|
|
"epoch": 0.195,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.561298847198486,
|
|
"learning_rate": 8.1e-07,
|
|
"loss": -0.06235164776444435,
|
|
"num_tokens": 325290.0,
|
|
"reward": -7.425000190734863,
|
|
"reward_std": 53.701602935791016,
|
|
"rewards/json_format_reward/mean": 0.07500000298023224,
|
|
"rewards/json_format_reward/std": 0.0707106813788414,
|
|
"rewards/supergames_reward/mean": -7.5,
|
|
"rewards/supergames_reward/std": 53.652320861816406,
|
|
"step": 39,
|
|
"step_time": 6.753862089011818
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 370.0,
|
|
"completions/max_terminated_length": 370.0,
|
|
"completions/mean_length": 187.625,
|
|
"completions/mean_terminated_length": 187.625,
|
|
"completions/min_length": 60.0,
|
|
"completions/min_terminated_length": 60.0,
|
|
"entropy": 0.3950234353542328,
|
|
"epoch": 0.2,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.5988712310791016,
|
|
"learning_rate": 8.05e-07,
|
|
"loss": -0.1381440907716751,
|
|
"num_tokens": 335119.0,
|
|
"reward": -33.68375015258789,
|
|
"reward_std": 42.069435119628906,
|
|
"rewards/json_format_reward/mean": 0.10000000149011612,
|
|
"rewards/json_format_reward/std": 0.0,
|
|
"rewards/supergames_reward/mean": -33.783748626708984,
|
|
"rewards/supergames_reward/std": 42.069435119628906,
|
|
"step": 40,
|
|
"step_time": 13.718958162004128
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 200,
|
|
"num_input_tokens_seen": 335119,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 20,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 8,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|