575 lines
19 KiB
JSON
575 lines
19 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 6.666666666666667,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 200,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.8625,
|
||
|
|
"completions/max_length": 379.3,
|
||
|
|
"completions/max_terminated_length": 57.8,
|
||
|
|
"completions/mean_length": 362.7625,
|
||
|
|
"completions/mean_terminated_length": 46.4125,
|
||
|
|
"completions/min_length": 318.1,
|
||
|
|
"completions/min_terminated_length": 38.1,
|
||
|
|
"entropy": 0.5937434114515782,
|
||
|
|
"epoch": 0.3333333333333333,
|
||
|
|
"frac_reward_zero_std": 0.8,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.9280000000000002e-05,
|
||
|
|
"loss": -0.017474760115146638,
|
||
|
|
"num_tokens": 61847.0,
|
||
|
|
"reward": -0.0987912505865097,
|
||
|
|
"reward_std": 0.9303328216075897,
|
||
|
|
"rewards/reward_fn/mean": -0.0987912505865097,
|
||
|
|
"rewards/reward_fn/std": 0.9303328156471252,
|
||
|
|
"step": 10,
|
||
|
|
"step_time": 31.724731107300066
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 1.0,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 0.0,
|
||
|
|
"completions/mean_length": 400.0,
|
||
|
|
"completions/mean_terminated_length": 0.0,
|
||
|
|
"completions/min_length": 400.0,
|
||
|
|
"completions/min_terminated_length": 0.0,
|
||
|
|
"entropy": 0.6593769532628357,
|
||
|
|
"epoch": 0.6666666666666666,
|
||
|
|
"frac_reward_zero_std": 1.0,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.8480000000000003e-05,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 126843.0,
|
||
|
|
"reward": 0.048249998688697816,
|
||
|
|
"reward_std": 0.8191121600568294,
|
||
|
|
"rewards/reward_fn/mean": 0.048249998688697816,
|
||
|
|
"rewards/reward_fn/std": 0.8191121838986873,
|
||
|
|
"step": 20,
|
||
|
|
"step_time": 33.554111711299946
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.975,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 49.7,
|
||
|
|
"completions/mean_length": 396.2125,
|
||
|
|
"completions/mean_terminated_length": 49.7,
|
||
|
|
"completions/min_length": 369.7,
|
||
|
|
"completions/min_terminated_length": 49.7,
|
||
|
|
"entropy": 0.5578590292367153,
|
||
|
|
"epoch": 1.0,
|
||
|
|
"frac_reward_zero_std": 0.9,
|
||
|
|
"grad_norm": 1.8046875,
|
||
|
|
"learning_rate": 1.768e-05,
|
||
|
|
"loss": -0.005067326501011849,
|
||
|
|
"num_tokens": 191538.0,
|
||
|
|
"reward": 0.11049998998641967,
|
||
|
|
"reward_std": 0.711869715154171,
|
||
|
|
"rewards/reward_fn/mean": 0.11049998998641967,
|
||
|
|
"rewards/reward_fn/std": 0.7118697345256806,
|
||
|
|
"step": 30,
|
||
|
|
"step_time": 33.49379607419994
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 1.0,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 0.0,
|
||
|
|
"completions/mean_length": 400.0,
|
||
|
|
"completions/mean_terminated_length": 0.0,
|
||
|
|
"completions/min_length": 400.0,
|
||
|
|
"completions/min_terminated_length": 0.0,
|
||
|
|
"entropy": 0.004339599603554234,
|
||
|
|
"epoch": 1.3333333333333333,
|
||
|
|
"frac_reward_zero_std": 0.95,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.688e-05,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 256536.0,
|
||
|
|
"reward": 0.07887499779462814,
|
||
|
|
"reward_std": 0.8938172444701195,
|
||
|
|
"rewards/reward_fn/mean": 0.07887499779462814,
|
||
|
|
"rewards/reward_fn/std": 0.8938172534108162,
|
||
|
|
"step": 40,
|
||
|
|
"step_time": 33.4393088197
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.9875,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 0.9,
|
||
|
|
"completions/mean_length": 395.1125,
|
||
|
|
"completions/mean_terminated_length": 0.9,
|
||
|
|
"completions/min_length": 360.9,
|
||
|
|
"completions/min_terminated_length": 0.9,
|
||
|
|
"entropy": 0.004430754791246727,
|
||
|
|
"epoch": 1.6666666666666665,
|
||
|
|
"frac_reward_zero_std": 0.975,
|
||
|
|
"grad_norm": 0.384765625,
|
||
|
|
"learning_rate": 1.6080000000000002e-05,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 321071.0,
|
||
|
|
"reward": -0.16437500417232515,
|
||
|
|
"reward_std": 1.063568216562271,
|
||
|
|
"rewards/reward_fn/mean": -0.16437500417232515,
|
||
|
|
"rewards/reward_fn/std": 1.0635682106018067,
|
||
|
|
"step": 50,
|
||
|
|
"step_time": 33.5646041849001
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.9625,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 3.7,
|
||
|
|
"completions/mean_length": 385.4625,
|
||
|
|
"completions/mean_terminated_length": 3.7,
|
||
|
|
"completions/min_length": 283.7,
|
||
|
|
"completions/min_terminated_length": 3.7,
|
||
|
|
"entropy": 0.09348616125644185,
|
||
|
|
"epoch": 2.0,
|
||
|
|
"frac_reward_zero_std": 0.925,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.5280000000000003e-05,
|
||
|
|
"loss": 0.009755914658308029,
|
||
|
|
"num_tokens": 384804.0,
|
||
|
|
"reward": -0.16562500596046448,
|
||
|
|
"reward_std": 0.9360396310687065,
|
||
|
|
"rewards/reward_fn/mean": -0.16562500596046448,
|
||
|
|
"rewards/reward_fn/std": 0.9360396608710289,
|
||
|
|
"step": 60,
|
||
|
|
"step_time": 33.48749011520003
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.8625,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 44.3,
|
||
|
|
"completions/mean_length": 351.7625,
|
||
|
|
"completions/mean_terminated_length": 42.26666679382324,
|
||
|
|
"completions/min_length": 159.8,
|
||
|
|
"completions/min_terminated_length": 39.8,
|
||
|
|
"entropy": 0.6013902719132602,
|
||
|
|
"epoch": 2.3333333333333335,
|
||
|
|
"frac_reward_zero_std": 0.925,
|
||
|
|
"grad_norm": 2.40625,
|
||
|
|
"learning_rate": 1.448e-05,
|
||
|
|
"loss": -0.011668374389410019,
|
||
|
|
"num_tokens": 445825.0,
|
||
|
|
"reward": 0.08912500143051147,
|
||
|
|
"reward_std": 0.9901719689369202,
|
||
|
|
"rewards/reward_fn/mean": 0.08912500143051147,
|
||
|
|
"rewards/reward_fn/std": 0.9901719927787781,
|
||
|
|
"step": 70,
|
||
|
|
"step_time": 33.48351585430014
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.9,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 36.8,
|
||
|
|
"completions/mean_length": 365.4875,
|
||
|
|
"completions/mean_terminated_length": 34.75,
|
||
|
|
"completions/min_length": 192.7,
|
||
|
|
"completions/min_terminated_length": 32.7,
|
||
|
|
"entropy": 0.7647990029305219,
|
||
|
|
"epoch": 2.6666666666666665,
|
||
|
|
"frac_reward_zero_std": 0.975,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.3680000000000003e-05,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 507944.0,
|
||
|
|
"reward": -0.14375001043081284,
|
||
|
|
"reward_std": 0.6796593680977822,
|
||
|
|
"rewards/reward_fn/mean": -0.14375001043081284,
|
||
|
|
"rewards/reward_fn/std": 0.6796593815088272,
|
||
|
|
"step": 80,
|
||
|
|
"step_time": 33.443731949500034
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.95,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 38.7,
|
||
|
|
"completions/mean_length": 386.45,
|
||
|
|
"completions/mean_terminated_length": 35.4,
|
||
|
|
"completions/min_length": 312.1,
|
||
|
|
"completions/min_terminated_length": 32.1,
|
||
|
|
"entropy": 0.817728553712368,
|
||
|
|
"epoch": 3.0,
|
||
|
|
"frac_reward_zero_std": 1.0,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.2880000000000002e-05,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 571920.0,
|
||
|
|
"reward": -0.09100000560283661,
|
||
|
|
"reward_std": 0.7491292104125022,
|
||
|
|
"rewards/reward_fn/mean": -0.09100000560283661,
|
||
|
|
"rewards/reward_fn/std": 0.7491292074322701,
|
||
|
|
"step": 90,
|
||
|
|
"step_time": 33.4239154713001
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.95,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 52.1,
|
||
|
|
"completions/mean_length": 386.5125,
|
||
|
|
"completions/mean_terminated_length": 52.1,
|
||
|
|
"completions/min_length": 292.1,
|
||
|
|
"completions/min_terminated_length": 52.1,
|
||
|
|
"entropy": 0.7871452756226063,
|
||
|
|
"epoch": 3.3333333333333335,
|
||
|
|
"frac_reward_zero_std": 1.0,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.2080000000000001e-05,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 635831.0,
|
||
|
|
"reward": -0.11849999725818634,
|
||
|
|
"reward_std": 0.9780218183994294,
|
||
|
|
"rewards/reward_fn/mean": -0.11849999725818634,
|
||
|
|
"rewards/reward_fn/std": 0.9780218482017518,
|
||
|
|
"step": 100,
|
||
|
|
"step_time": 33.44708331829988
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.95,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 75.3,
|
||
|
|
"completions/mean_length": 389.775,
|
||
|
|
"completions/mean_terminated_length": 65.95,
|
||
|
|
"completions/min_length": 336.6,
|
||
|
|
"completions/min_terminated_length": 56.6,
|
||
|
|
"entropy": 0.8245016686618328,
|
||
|
|
"epoch": 3.6666666666666665,
|
||
|
|
"frac_reward_zero_std": 1.0,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.128e-05,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 699917.0,
|
||
|
|
"reward": 0.06824999898672104,
|
||
|
|
"reward_std": 0.8597677208483219,
|
||
|
|
"rewards/reward_fn/mean": 0.06824999898672104,
|
||
|
|
"rewards/reward_fn/std": 0.8597677327692509,
|
||
|
|
"step": 110,
|
||
|
|
"step_time": 33.43369797569976
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.9875,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 29.0,
|
||
|
|
"completions/mean_length": 398.625,
|
||
|
|
"completions/mean_terminated_length": 29.0,
|
||
|
|
"completions/min_length": 389.0,
|
||
|
|
"completions/min_terminated_length": 29.0,
|
||
|
|
"entropy": 0.8096485503017903,
|
||
|
|
"epoch": 4.0,
|
||
|
|
"frac_reward_zero_std": 1.0,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 1.0480000000000001e-05,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 764733.0,
|
||
|
|
"reward": -0.05875001698732376,
|
||
|
|
"reward_std": 0.7073742881417274,
|
||
|
|
"rewards/reward_fn/mean": -0.05875001698732376,
|
||
|
|
"rewards/reward_fn/std": 0.7073742985725403,
|
||
|
|
"step": 120,
|
||
|
|
"step_time": 33.47654410239984
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.9625,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 51.8,
|
||
|
|
"completions/mean_length": 391.475,
|
||
|
|
"completions/mean_terminated_length": 51.8,
|
||
|
|
"completions/min_length": 331.8,
|
||
|
|
"completions/min_terminated_length": 51.8,
|
||
|
|
"entropy": 0.8178432904183864,
|
||
|
|
"epoch": 4.333333333333333,
|
||
|
|
"frac_reward_zero_std": 0.975,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 9.68e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 828855.0,
|
||
|
|
"reward": -0.017000006139278413,
|
||
|
|
"reward_std": 0.7560826383531094,
|
||
|
|
"rewards/reward_fn/mean": -0.017000006139278413,
|
||
|
|
"rewards/reward_fn/std": 0.7560826435685157,
|
||
|
|
"step": 130,
|
||
|
|
"step_time": 33.514995820599964
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.9375,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 46.2,
|
||
|
|
"completions/mean_length": 380.775,
|
||
|
|
"completions/mean_terminated_length": 46.2,
|
||
|
|
"completions/min_length": 246.2,
|
||
|
|
"completions/min_terminated_length": 46.2,
|
||
|
|
"entropy": 0.8854692216962576,
|
||
|
|
"epoch": 4.666666666666667,
|
||
|
|
"frac_reward_zero_std": 0.95,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 8.880000000000001e-06,
|
||
|
|
"loss": -0.012354153394699096,
|
||
|
|
"num_tokens": 892373.0,
|
||
|
|
"reward": -0.13475000411272048,
|
||
|
|
"reward_std": 0.9627669990062714,
|
||
|
|
"rewards/reward_fn/mean": -0.13475000411272048,
|
||
|
|
"rewards/reward_fn/std": 0.9627670347690582,
|
||
|
|
"step": 140,
|
||
|
|
"step_time": 33.50577180429955
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.975,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 32.3,
|
||
|
|
"completions/mean_length": 394.275,
|
||
|
|
"completions/mean_terminated_length": 17.1,
|
||
|
|
"completions/min_length": 361.9,
|
||
|
|
"completions/min_terminated_length": 1.9,
|
||
|
|
"entropy": 0.8237141810357571,
|
||
|
|
"epoch": 5.0,
|
||
|
|
"frac_reward_zero_std": 1.0,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 8.08e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 956875.0,
|
||
|
|
"reward": -0.030000001192092896,
|
||
|
|
"reward_std": 0.7297740295529366,
|
||
|
|
"rewards/reward_fn/mean": -0.030000001192092896,
|
||
|
|
"rewards/reward_fn/std": 0.7297740370035172,
|
||
|
|
"step": 150,
|
||
|
|
"step_time": 33.46883516300004
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.975,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 38.5,
|
||
|
|
"completions/mean_length": 394.8125,
|
||
|
|
"completions/mean_terminated_length": 38.5,
|
||
|
|
"completions/min_length": 358.5,
|
||
|
|
"completions/min_terminated_length": 38.5,
|
||
|
|
"entropy": 0.9815139673650265,
|
||
|
|
"epoch": 5.333333333333333,
|
||
|
|
"frac_reward_zero_std": 1.0,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 7.280000000000001e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 1021402.0,
|
||
|
|
"reward": -0.06250000596046448,
|
||
|
|
"reward_std": 0.9604951858520507,
|
||
|
|
"rewards/reward_fn/mean": -0.06250000596046448,
|
||
|
|
"rewards/reward_fn/std": 0.9604952037334442,
|
||
|
|
"step": 160,
|
||
|
|
"step_time": 33.41488298210034
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.975,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 21.2,
|
||
|
|
"completions/mean_length": 392.65,
|
||
|
|
"completions/mean_terminated_length": 21.2,
|
||
|
|
"completions/min_length": 341.2,
|
||
|
|
"completions/min_terminated_length": 21.2,
|
||
|
|
"entropy": 0.9070465706288815,
|
||
|
|
"epoch": 5.666666666666667,
|
||
|
|
"frac_reward_zero_std": 0.975,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 6.480000000000001e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 1085632.0,
|
||
|
|
"reward": -0.1513125091791153,
|
||
|
|
"reward_std": 0.855255764722824,
|
||
|
|
"rewards/reward_fn/mean": -0.1513125091791153,
|
||
|
|
"rewards/reward_fn/std": 0.8552557826042175,
|
||
|
|
"step": 170,
|
||
|
|
"step_time": 33.4777201513999
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 1.0,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 0.0,
|
||
|
|
"completions/mean_length": 400.0,
|
||
|
|
"completions/mean_terminated_length": 0.0,
|
||
|
|
"completions/min_length": 400.0,
|
||
|
|
"completions/min_terminated_length": 0.0,
|
||
|
|
"entropy": 0.9226562466472388,
|
||
|
|
"epoch": 6.0,
|
||
|
|
"frac_reward_zero_std": 1.0,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 5.68e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 1150692.0,
|
||
|
|
"reward": 0.08200000673532486,
|
||
|
|
"reward_std": 0.9346169531345367,
|
||
|
|
"rewards/reward_fn/mean": 0.08200000673532486,
|
||
|
|
"rewards/reward_fn/std": 0.9346169650554657,
|
||
|
|
"step": 180,
|
||
|
|
"step_time": 33.47372673979989
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.975,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 35.5,
|
||
|
|
"completions/mean_length": 394.4375,
|
||
|
|
"completions/mean_terminated_length": 35.5,
|
||
|
|
"completions/min_length": 355.5,
|
||
|
|
"completions/min_terminated_length": 35.5,
|
||
|
|
"entropy": 0.9981254413723946,
|
||
|
|
"epoch": 6.333333333333333,
|
||
|
|
"frac_reward_zero_std": 0.975,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.880000000000001e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 1215133.0,
|
||
|
|
"reward": -0.08775000870227814,
|
||
|
|
"reward_std": 0.8883017361164093,
|
||
|
|
"rewards/reward_fn/mean": -0.08775000870227814,
|
||
|
|
"rewards/reward_fn/std": 0.8883017897605896,
|
||
|
|
"step": 190,
|
||
|
|
"step_time": 33.41868621859985
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.9875,
|
||
|
|
"completions/max_length": 400.0,
|
||
|
|
"completions/max_terminated_length": 30.4,
|
||
|
|
"completions/mean_length": 398.8,
|
||
|
|
"completions/mean_terminated_length": 30.4,
|
||
|
|
"completions/min_length": 390.4,
|
||
|
|
"completions/min_terminated_length": 30.4,
|
||
|
|
"entropy": 0.9789559677243233,
|
||
|
|
"epoch": 6.666666666666667,
|
||
|
|
"frac_reward_zero_std": 1.0,
|
||
|
|
"grad_norm": 0.0,
|
||
|
|
"learning_rate": 4.08e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 1279879.0,
|
||
|
|
"reward": -0.03525000065565109,
|
||
|
|
"reward_std": 0.8935358546674251,
|
||
|
|
"rewards/reward_fn/mean": -0.03525000065565109,
|
||
|
|
"rewards/reward_fn/std": 0.8935358554124833,
|
||
|
|
"step": 200,
|
||
|
|
"step_time": 33.424166100800086
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 10,
|
||
|
|
"max_steps": 250,
|
||
|
|
"num_input_tokens_seen": 1279879,
|
||
|
|
"num_train_epochs": 9,
|
||
|
|
"save_steps": 100,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": false
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 2,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|