8285 lines
298 KiB
JSON
8285 lines
298 KiB
JSON
{
|
|
"best_global_step": null,
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.20833333333333334,
|
|
"eval_steps": 500,
|
|
"global_step": 250,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0008333333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.0297584533691406,
|
|
"kl": 0.007555788848549128,
|
|
"learning_rate": 0.0,
|
|
"loss": 0.0003,
|
|
"num_tokens": 2438.0,
|
|
"reward": 0.5746050477027893,
|
|
"reward_std": 0.7637181282043457,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.16745585203170776,
|
|
"rewards/belief_accuracy/std": 0.03546026349067688,
|
|
"rewards/env_reward/mean": 0.014825001358985901,
|
|
"rewards/env_reward/std": 0.5229976177215576,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 1
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.0,
|
|
"completions/mean_terminated_length": 9.333333969116211,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0016666666666666668,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 487.28997802734375,
|
|
"kl": 0.36721258889883757,
|
|
"learning_rate": 1.25e-06,
|
|
"loss": 0.0147,
|
|
"num_tokens": 4754.0,
|
|
"reward": 1.6277785301208496,
|
|
"reward_std": 0.3852787911891937,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.37001365423202515,
|
|
"rewards/belief_accuracy/std": 0.06095712259411812,
|
|
"rewards/env_reward/mean": 0.3118250072002411,
|
|
"rewards/env_reward/std": 0.26973089575767517,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 2
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 12.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 12.0,
|
|
"completions/max_terminated_length": 12.0,
|
|
"completions/mean_length": 10.25,
|
|
"completions/mean_terminated_length": 10.25,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0025,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 11.092467308044434,
|
|
"kl": 0.014069308177568018,
|
|
"learning_rate": 2.5e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 7195.0,
|
|
"reward": -0.9344245195388794,
|
|
"reward_std": 2.935027599334717,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.03502512723207474,
|
|
"rewards/belief_accuracy/std": 0.24171829223632812,
|
|
"rewards/env_reward/mean": -0.6929999589920044,
|
|
"rewards/env_reward/std": 1.5391119718551636,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 3
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 18.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 18.0,
|
|
"completions/max_terminated_length": 18.0,
|
|
"completions/mean_length": 11.5,
|
|
"completions/mean_terminated_length": 11.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0033333333333333335,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 70.31130981445312,
|
|
"kl": 0.16654532926622778,
|
|
"learning_rate": 3.75e-06,
|
|
"loss": 0.0067,
|
|
"num_tokens": 9641.0,
|
|
"reward": 0.40939950942993164,
|
|
"reward_std": 0.6117480397224426,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.04012066870927811,
|
|
"rewards/belief_accuracy/std": 0.20485758781433105,
|
|
"rewards/env_reward/mean": 0.16352500021457672,
|
|
"rewards/env_reward/std": 0.12513433396816254,
|
|
"rewards/format_valid/mean": 0.875,
|
|
"rewards/format_valid/std": 0.25,
|
|
"step": 4
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 11.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 11.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.004166666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.75938606262207,
|
|
"kl": 0.005745900256442837,
|
|
"learning_rate": 5e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 12081.0,
|
|
"reward": 1.2169301509857178,
|
|
"reward_std": 0.5240868330001831,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2165767103433609,
|
|
"rewards/belief_accuracy/std": 0.21178469061851501,
|
|
"rewards/env_reward/mean": 0.3447999954223633,
|
|
"rewards/env_reward/std": 0.1597922146320343,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 5
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.25,
|
|
"completions/mean_terminated_length": 9.25,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.005,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.055194854736328,
|
|
"kl": 0.0013933435111539438,
|
|
"learning_rate": 6.25e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 14378.0,
|
|
"reward": 0.8722533583641052,
|
|
"reward_std": 0.5229519605636597,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.17082196474075317,
|
|
"rewards/belief_accuracy/std": 0.06307334452867508,
|
|
"rewards/env_reward/mean": 0.20652499794960022,
|
|
"rewards/env_reward/std": 0.31724998354911804,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 6
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.005833333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.596080780029297,
|
|
"kl": 0.003983344242442399,
|
|
"learning_rate": 7.5e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 16817.0,
|
|
"reward": 0.3588276505470276,
|
|
"reward_std": 0.5061821937561035,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.07134255766868591,
|
|
"rewards/belief_accuracy/std": 0.08802108466625214,
|
|
"rewards/env_reward/mean": 0.06319999694824219,
|
|
"rewards/env_reward/std": 0.1783815622329712,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 7
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.25,
|
|
"completions/mean_terminated_length": 9.25,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.006666666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.14266300201416,
|
|
"kl": 0.0045957728871144354,
|
|
"learning_rate": 8.75e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 19254.0,
|
|
"reward": 0.9281585216522217,
|
|
"reward_std": 0.641987144947052,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.25081950426101685,
|
|
"rewards/belief_accuracy/std": 0.09036833792924881,
|
|
"rewards/env_reward/mean": 0.08379998803138733,
|
|
"rewards/env_reward/std": 0.4169001281261444,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 8
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 11.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 11.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0075,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.244797229766846,
|
|
"kl": 0.004008802643511444,
|
|
"learning_rate": 1e-05,
|
|
"loss": 0.0002,
|
|
"num_tokens": 21694.0,
|
|
"reward": -0.8402208089828491,
|
|
"reward_std": 2.993889331817627,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.08231388032436371,
|
|
"rewards/belief_accuracy/std": 0.22892379760742188,
|
|
"rewards/env_reward/mean": -0.7247750163078308,
|
|
"rewards/env_reward/std": 1.522688627243042,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 9
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 25.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 25.0,
|
|
"completions/max_terminated_length": 25.0,
|
|
"completions/mean_length": 14.25,
|
|
"completions/mean_terminated_length": 14.25,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.008333333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 12.427257537841797,
|
|
"kl": 0.028211204218678176,
|
|
"learning_rate": 1.125e-05,
|
|
"loss": 0.0011,
|
|
"num_tokens": 24151.0,
|
|
"reward": 0.732832670211792,
|
|
"reward_std": 0.2891167104244232,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.1579858809709549,
|
|
"rewards/belief_accuracy/std": 0.0883205309510231,
|
|
"rewards/env_reward/mean": 0.1392499953508377,
|
|
"rewards/env_reward/std": 0.016723934561014175,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.009166666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.1559700965881348,
|
|
"kl": 0.005673486019077245,
|
|
"learning_rate": 1.25e-05,
|
|
"loss": 0.0002,
|
|
"num_tokens": 25878.0,
|
|
"reward": -0.2948700189590454,
|
|
"reward_std": 3.3204774856567383,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.16303499042987823,
|
|
"rewards/belief_accuracy/std": 0.24976226687431335,
|
|
"rewards/env_reward/mean": -0.5226500034332275,
|
|
"rewards/env_reward/std": 1.6547037363052368,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 11
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.01,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.764303207397461,
|
|
"kl": 0.004842391535930801,
|
|
"learning_rate": 1.3750000000000002e-05,
|
|
"loss": 0.0002,
|
|
"num_tokens": 27605.0,
|
|
"reward": 0.6823896169662476,
|
|
"reward_std": 0.017910229042172432,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.22252154350280762,
|
|
"rewards/belief_accuracy/std": 0.036789700388908386,
|
|
"rewards/env_reward/mean": -0.023450002074241638,
|
|
"rewards/env_reward/std": 0.07271481305360794,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 12
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 25.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 25.0,
|
|
"completions/max_terminated_length": 25.0,
|
|
"completions/mean_length": 14.0,
|
|
"completions/mean_terminated_length": 14.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.010833333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.114142894744873,
|
|
"kl": 0.004877378873061389,
|
|
"learning_rate": 1.5e-05,
|
|
"loss": 0.0002,
|
|
"num_tokens": 30061.0,
|
|
"reward": 0.8373932242393494,
|
|
"reward_std": 0.4389983117580414,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.23391437530517578,
|
|
"rewards/belief_accuracy/std": 0.1463327705860138,
|
|
"rewards/env_reward/mean": 0.057100001722574234,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 13
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.011666666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.0929784774780273,
|
|
"kl": 0.0014771804580959724,
|
|
"learning_rate": 1.6250000000000002e-05,
|
|
"loss": 0.0001,
|
|
"num_tokens": 32501.0,
|
|
"reward": 0.20685642957687378,
|
|
"reward_std": 0.6949325203895569,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.19772300124168396,
|
|
"rewards/belief_accuracy/std": 0.1775362491607666,
|
|
"rewards/env_reward/mean": -0.29087501764297485,
|
|
"rewards/env_reward/std": 0.7713783979415894,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 14
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0125,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.996659755706787,
|
|
"kl": 0.007067542013828643,
|
|
"learning_rate": 1.75e-05,
|
|
"loss": 0.0003,
|
|
"num_tokens": 34939.0,
|
|
"reward": 0.9863791465759277,
|
|
"reward_std": 0.1628992110490799,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.18266388773918152,
|
|
"rewards/belief_accuracy/std": 0.0645347312092781,
|
|
"rewards/env_reward/mean": 0.25892502069473267,
|
|
"rewards/env_reward/std": 0.1236146092414856,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 15
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.013333333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.10272216796875,
|
|
"kl": 0.004334542165452149,
|
|
"learning_rate": 1.8750000000000002e-05,
|
|
"loss": 0.0002,
|
|
"num_tokens": 37377.0,
|
|
"reward": 0.9267517924308777,
|
|
"reward_std": 0.1699640452861786,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.16352561116218567,
|
|
"rewards/belief_accuracy/std": 0.09197874367237091,
|
|
"rewards/env_reward/mean": 0.2574499845504761,
|
|
"rewards/env_reward/std": 0.2165336161851883,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 16
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.014166666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.821376800537109,
|
|
"kl": 0.004073493357282132,
|
|
"learning_rate": 2e-05,
|
|
"loss": 0.0002,
|
|
"num_tokens": 39815.0,
|
|
"reward": -1.199587106704712,
|
|
"reward_std": 2.7017223834991455,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.08026264607906342,
|
|
"rewards/belief_accuracy/std": 0.1924673169851303,
|
|
"rewards/env_reward/mean": -0.9602500200271606,
|
|
"rewards/env_reward/std": 1.3646483421325684,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 17
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.015,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.039498805999756,
|
|
"kl": 0.003853246627841145,
|
|
"learning_rate": 2.125e-05,
|
|
"loss": 0.0002,
|
|
"num_tokens": 42253.0,
|
|
"reward": 1.1887301206588745,
|
|
"reward_std": 0.5090645551681519,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.1981642246246338,
|
|
"rewards/belief_accuracy/std": 0.08274012058973312,
|
|
"rewards/env_reward/mean": 0.3628250062465668,
|
|
"rewards/env_reward/std": 0.42794182896614075,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 18
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.015833333333333335,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 8.75765609741211,
|
|
"kl": 0.002594503777800128,
|
|
"learning_rate": 2.25e-05,
|
|
"loss": 0.0001,
|
|
"num_tokens": 44693.0,
|
|
"reward": 1.2963204383850098,
|
|
"reward_std": 1.6085225343704224,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.1160651445388794,
|
|
"rewards/belief_accuracy/std": 0.10867781937122345,
|
|
"rewards/env_reward/mean": 0.5987499952316284,
|
|
"rewards/env_reward/std": 0.9246196746826172,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 19
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.016666666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.9152791500091553,
|
|
"kl": 0.0029762745389234624,
|
|
"learning_rate": 2.375e-05,
|
|
"loss": 0.0001,
|
|
"num_tokens": 46624.0,
|
|
"reward": 0.6017731428146362,
|
|
"reward_std": 0.15383398532867432,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.09383687376976013,
|
|
"rewards/belief_accuracy/std": 0.03703703731298447,
|
|
"rewards/env_reward/mean": 0.18017500638961792,
|
|
"rewards/env_reward/std": 0.15538449585437775,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 11.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 11.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 10.5,
|
|
"completions/mean_terminated_length": 10.5,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.0175,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.0644450187683105,
|
|
"kl": 0.00437403135583736,
|
|
"learning_rate": 2.5e-05,
|
|
"loss": 0.0002,
|
|
"num_tokens": 49066.0,
|
|
"reward": -0.5255992412567139,
|
|
"reward_std": 3.1553311347961426,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.122025266289711,
|
|
"rewards/belief_accuracy/std": 0.2217501848936081,
|
|
"rewards/env_reward/mean": -0.5944499969482422,
|
|
"rewards/env_reward/std": 1.6077580451965332,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 21
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.018333333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.2595951557159424,
|
|
"kl": 0.0019797176646534353,
|
|
"learning_rate": 2.625e-05,
|
|
"loss": 0.0001,
|
|
"num_tokens": 51505.0,
|
|
"reward": 0.4400880038738251,
|
|
"reward_std": 0.5773349404335022,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.17279182374477386,
|
|
"rewards/belief_accuracy/std": 0.07431290298700333,
|
|
"rewards/env_reward/mean": -0.08552499860525131,
|
|
"rewards/env_reward/std": 0.2797339856624603,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 22
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.25,
|
|
"completions/mean_terminated_length": 9.666666984558105,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.019166666666666665,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 736.62890625,
|
|
"kl": 3.031015553511679,
|
|
"learning_rate": 2.7500000000000004e-05,
|
|
"loss": 0.1212,
|
|
"num_tokens": 53966.0,
|
|
"reward": 1.050592064857483,
|
|
"reward_std": 0.2744295299053192,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2777556777000427,
|
|
"rewards/belief_accuracy/std": 0.10405799001455307,
|
|
"rewards/env_reward/mean": 0.1115499958395958,
|
|
"rewards/env_reward/std": 0.04209999740123749,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 23
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 11.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 11.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 10.25,
|
|
"completions/mean_terminated_length": 10.25,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.02,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.14186954498291,
|
|
"kl": 0.01767307601403445,
|
|
"learning_rate": 2.8749999999999997e-05,
|
|
"loss": 0.0007,
|
|
"num_tokens": 56407.0,
|
|
"reward": -0.10755039006471634,
|
|
"reward_std": 1.0050357580184937,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.16075819730758667,
|
|
"rewards/belief_accuracy/std": 0.09638070315122604,
|
|
"rewards/env_reward/mean": -0.4265500009059906,
|
|
"rewards/env_reward/std": 0.7103733420372009,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 24
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.020833333333333332,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.760839462280273,
|
|
"kl": 0.004387545719509944,
|
|
"learning_rate": 3e-05,
|
|
"loss": 0.0002,
|
|
"num_tokens": 58846.0,
|
|
"reward": -0.6063781976699829,
|
|
"reward_std": 3.0987460613250732,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.08613643050193787,
|
|
"rewards/belief_accuracy/std": 0.20087628066539764,
|
|
"rewards/env_reward/mean": -0.5765249729156494,
|
|
"rewards/env_reward/std": 1.6289700269699097,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 25
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.021666666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.131656646728516,
|
|
"kl": 0.006817424291511998,
|
|
"learning_rate": 3.125e-05,
|
|
"loss": 0.0003,
|
|
"num_tokens": 61285.0,
|
|
"reward": 1.4750633239746094,
|
|
"reward_std": 0.6433582305908203,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.22258360683918,
|
|
"rewards/belief_accuracy/std": 0.0710342675447464,
|
|
"rewards/env_reward/mean": 0.5048750042915344,
|
|
"rewards/env_reward/std": 0.30701497197151184,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 26
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0225,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.655338764190674,
|
|
"kl": 0.011261592793744057,
|
|
"learning_rate": 3.2500000000000004e-05,
|
|
"loss": 0.0005,
|
|
"num_tokens": 63724.0,
|
|
"reward": 1.4208177328109741,
|
|
"reward_std": 1.19350266456604,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.21906425058841705,
|
|
"rewards/belief_accuracy/std": 0.18924623727798462,
|
|
"rewards/env_reward/mean": 0.47574999928474426,
|
|
"rewards/env_reward/std": 0.4241226613521576,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 27
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 15.25,
|
|
"completions/mean_terminated_length": 9.666666984558105,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.023333333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 35.8610725402832,
|
|
"kl": 4.056387651711702,
|
|
"learning_rate": 3.375000000000001e-05,
|
|
"loss": 0.1623,
|
|
"num_tokens": 65857.0,
|
|
"reward": 0.2565092444419861,
|
|
"reward_std": 0.9514979124069214,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.22687393426895142,
|
|
"rewards/belief_accuracy/std": 0.07754969596862793,
|
|
"rewards/env_reward/mean": -0.31607499718666077,
|
|
"rewards/env_reward/std": 0.7444266080856323,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 28
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.024166666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.129375696182251,
|
|
"kl": 0.06665351914125495,
|
|
"learning_rate": 3.5e-05,
|
|
"loss": 0.0027,
|
|
"num_tokens": 68295.0,
|
|
"reward": 1.5676257610321045,
|
|
"reward_std": 0.5401614904403687,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.23782525956630707,
|
|
"rewards/belief_accuracy/std": 0.16049730777740479,
|
|
"rewards/env_reward/mean": 0.5361000299453735,
|
|
"rewards/env_reward/std": 0.2634325325489044,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 29
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.25,
|
|
"completions/mean_terminated_length": 9.25,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.025,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 7.313047885894775,
|
|
"kl": 0.09575654342916096,
|
|
"learning_rate": 3.625e-05,
|
|
"loss": 0.0038,
|
|
"num_tokens": 70592.0,
|
|
"reward": 1.385237693786621,
|
|
"reward_std": 0.23877514898777008,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.25660422444343567,
|
|
"rewards/belief_accuracy/std": 0.030240608379244804,
|
|
"rewards/env_reward/mean": 0.3769500255584717,
|
|
"rewards/env_reward/std": 0.1939132809638977,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 15.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 15.0,
|
|
"completions/max_terminated_length": 15.0,
|
|
"completions/mean_length": 11.0,
|
|
"completions/mean_terminated_length": 11.0,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.025833333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 302.0821838378906,
|
|
"kl": 0.17512649775017053,
|
|
"learning_rate": 3.7500000000000003e-05,
|
|
"loss": 0.007,
|
|
"num_tokens": 73036.0,
|
|
"reward": -0.8282430171966553,
|
|
"reward_std": 2.9490480422973633,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.08568151295185089,
|
|
"rewards/belief_accuracy/std": 0.19252440333366394,
|
|
"rewards/env_reward/mean": -0.7235249876976013,
|
|
"rewards/env_reward/std": 1.5176500082015991,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 31
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.5,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.02666666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 78.20838165283203,
|
|
"kl": 9.407063644379377,
|
|
"learning_rate": 3.875e-05,
|
|
"loss": 0.3763,
|
|
"num_tokens": 75498.0,
|
|
"reward": -1.4438053369522095,
|
|
"reward_std": 2.5607211589813232,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": -0.07945596426725388,
|
|
"rewards/belief_accuracy/std": 0.0959252119064331,
|
|
"rewards/env_reward/mean": -0.8036249876022339,
|
|
"rewards/env_reward/std": 1.5019314289093018,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 32
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0275,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.190960168838501,
|
|
"kl": 0.06879310176009312,
|
|
"learning_rate": 4e-05,
|
|
"loss": 0.0028,
|
|
"num_tokens": 77225.0,
|
|
"reward": 1.5751994848251343,
|
|
"reward_std": 0.5590165257453918,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.25149983167648315,
|
|
"rewards/belief_accuracy/std": 0.07094359397888184,
|
|
"rewards/env_reward/mean": 0.5138000249862671,
|
|
"rewards/env_reward/std": 0.23183931410312653,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 33
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 24.0,
|
|
"completions/mean_length": 18.75,
|
|
"completions/mean_terminated_length": 14.333333969116211,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.028333333333333332,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 28.640579223632812,
|
|
"kl": 0.42012222670018673,
|
|
"learning_rate": 4.125e-05,
|
|
"loss": 0.0168,
|
|
"num_tokens": 79700.0,
|
|
"reward": 0.3387056589126587,
|
|
"reward_std": 3.7822914123535156,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.2497768998146057,
|
|
"rewards/belief_accuracy/std": 0.30190309882164,
|
|
"rewards/env_reward/mean": -0.2737500071525574,
|
|
"rewards/env_reward/std": 1.855374813079834,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 34
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.029166666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.623953342437744,
|
|
"kl": 0.14891700155567378,
|
|
"learning_rate": 4.25e-05,
|
|
"loss": 0.006,
|
|
"num_tokens": 82138.0,
|
|
"reward": 0.45824581384658813,
|
|
"reward_std": 0.7270535230636597,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.25250691175460815,
|
|
"rewards/belief_accuracy/std": 0.1582489311695099,
|
|
"rewards/env_reward/mean": -0.23285000026226044,
|
|
"rewards/env_reward/std": 0.23289161920547485,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 35
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 11.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 11.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.03,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.508930683135986,
|
|
"kl": 0.2548879988025874,
|
|
"learning_rate": 4.375e-05,
|
|
"loss": 0.0102,
|
|
"num_tokens": 84578.0,
|
|
"reward": 0.4626871347427368,
|
|
"reward_std": 0.8850942254066467,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.24804989993572235,
|
|
"rewards/belief_accuracy/std": 0.12259609997272491,
|
|
"rewards/env_reward/mean": -0.22097501158714294,
|
|
"rewards/env_reward/std": 0.6560747623443604,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 36
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 11.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 11.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 10.25,
|
|
"completions/mean_terminated_length": 10.25,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.030833333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.627114295959473,
|
|
"kl": 0.132804719673004,
|
|
"learning_rate": 4.5e-05,
|
|
"loss": 0.0053,
|
|
"num_tokens": 87019.0,
|
|
"reward": -2.2696728706359863,
|
|
"reward_std": 3.4521307945251465,
|
|
"rewards/action_legal/mean": -0.5,
|
|
"rewards/action_legal/std": 0.5773502588272095,
|
|
"rewards/belief_accuracy/mean": -0.0548660047352314,
|
|
"rewards/belief_accuracy/std": 0.19056659936904907,
|
|
"rewards/env_reward/mean": -1.3700499534606934,
|
|
"rewards/env_reward/std": 1.8821041584014893,
|
|
"rewards/format_valid/mean": -0.5,
|
|
"rewards/format_valid/std": 1.7320507764816284,
|
|
"step": 37
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.03166666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.246164321899414,
|
|
"kl": 0.2298137085745111,
|
|
"learning_rate": 4.6250000000000006e-05,
|
|
"loss": 0.0092,
|
|
"num_tokens": 89457.0,
|
|
"reward": 1.1744983196258545,
|
|
"reward_std": 0.3355043828487396,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.29197028279304504,
|
|
"rewards/belief_accuracy/std": 0.12365185469388962,
|
|
"rewards/env_reward/mean": 0.1657249927520752,
|
|
"rewards/env_reward/std": 0.03991043195128441,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 38
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 15.25,
|
|
"completions/mean_terminated_length": 9.666666984558105,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0325,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 12.651698112487793,
|
|
"kl": 2.4256066400557756,
|
|
"learning_rate": 4.75e-05,
|
|
"loss": 0.097,
|
|
"num_tokens": 91918.0,
|
|
"reward": -0.7721501588821411,
|
|
"reward_std": 1.5534520149230957,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.13362497091293335,
|
|
"rewards/belief_accuracy/std": 0.1170458197593689,
|
|
"rewards/env_reward/mean": -0.8153500556945801,
|
|
"rewards/env_reward/std": 0.9287000298500061,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 39
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.03333333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 7.508624076843262,
|
|
"kl": 0.12869007809786126,
|
|
"learning_rate": 4.875e-05,
|
|
"loss": 0.0051,
|
|
"num_tokens": 93644.0,
|
|
"reward": 1.710510492324829,
|
|
"reward_std": 0.9325205683708191,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3421034812927246,
|
|
"rewards/belief_accuracy/std": 0.03489655256271362,
|
|
"rewards/env_reward/mean": 0.4228000044822693,
|
|
"rewards/env_reward/std": 0.6864694952964783,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.034166666666666665,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.437275409698486,
|
|
"kl": 0.23907954257447273,
|
|
"learning_rate": 5e-05,
|
|
"loss": 0.0096,
|
|
"num_tokens": 96082.0,
|
|
"reward": 1.1177518367767334,
|
|
"reward_std": 0.4233185946941376,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.1830797791481018,
|
|
"rewards/belief_accuracy/std": 0.0974152609705925,
|
|
"rewards/env_reward/mean": 0.345674991607666,
|
|
"rewards/env_reward/std": 0.2988019585609436,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 41
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 11.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 11.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.035,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.4014458656311035,
|
|
"kl": 0.1710746451281011,
|
|
"learning_rate": 4.986111111111111e-05,
|
|
"loss": 0.0068,
|
|
"num_tokens": 98197.0,
|
|
"reward": -0.028865892440080643,
|
|
"reward_std": 1.0444624423980713,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.05849887803196907,
|
|
"rewards/belief_accuracy/std": 0.0555555559694767,
|
|
"rewards/env_reward/mean": -0.16957500576972961,
|
|
"rewards/env_reward/std": 0.7329205870628357,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 42
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 9.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 9.0,
|
|
"completions/max_terminated_length": 9.0,
|
|
"completions/mean_length": 9.0,
|
|
"completions/mean_terminated_length": 9.0,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.035833333333333335,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.1398186683654785,
|
|
"kl": 0.044276596046984196,
|
|
"learning_rate": 4.972222222222223e-05,
|
|
"loss": 0.0018,
|
|
"num_tokens": 100633.0,
|
|
"reward": 0.06939443945884705,
|
|
"reward_std": 0.36057770252227783,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.18461482226848602,
|
|
"rewards/belief_accuracy/std": 0.1464960277080536,
|
|
"rewards/env_reward/mean": -0.3562999963760376,
|
|
"rewards/env_reward/std": 0.1712000072002411,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 43
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.25,
|
|
"completions/mean_terminated_length": 9.25,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.03666666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.34901237487793,
|
|
"kl": 0.3707070527598262,
|
|
"learning_rate": 4.958333333333334e-05,
|
|
"loss": 0.0148,
|
|
"num_tokens": 103070.0,
|
|
"reward": 0.9979052543640137,
|
|
"reward_std": 0.08536310493946075,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.28115594387054443,
|
|
"rewards/belief_accuracy/std": 0.12468524277210236,
|
|
"rewards/env_reward/mean": 0.06962499022483826,
|
|
"rewards/env_reward/std": 0.2918499708175659,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 44
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.25,
|
|
"completions/mean_terminated_length": 9.25,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0375,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.1270384788513184,
|
|
"kl": 0.27486721728928387,
|
|
"learning_rate": 4.9444444444444446e-05,
|
|
"loss": 0.011,
|
|
"num_tokens": 105367.0,
|
|
"reward": 1.3586945533752441,
|
|
"reward_std": 0.09579288214445114,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2644440233707428,
|
|
"rewards/belief_accuracy/std": 0.05163230374455452,
|
|
"rewards/env_reward/mean": 0.34357500076293945,
|
|
"rewards/env_reward/std": 0.07905000448226929,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 45
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 12.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 12.0,
|
|
"completions/max_terminated_length": 12.0,
|
|
"completions/mean_length": 10.5,
|
|
"completions/mean_terminated_length": 10.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.03833333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 11.598420143127441,
|
|
"kl": 0.35064559197053313,
|
|
"learning_rate": 4.930555555555556e-05,
|
|
"loss": 0.014,
|
|
"num_tokens": 107809.0,
|
|
"reward": -0.5861924886703491,
|
|
"reward_std": 3.1335580348968506,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.06579001247882843,
|
|
"rewards/belief_accuracy/std": 0.18941286206245422,
|
|
"rewards/env_reward/mean": -0.5223749876022339,
|
|
"rewards/env_reward/std": 1.6578657627105713,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 46
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.03916666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.6913492679595947,
|
|
"kl": 0.18329114187508821,
|
|
"learning_rate": 4.9166666666666665e-05,
|
|
"loss": 0.0073,
|
|
"num_tokens": 110248.0,
|
|
"reward": 1.4434185028076172,
|
|
"reward_std": 0.21208810806274414,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.302872896194458,
|
|
"rewards/belief_accuracy/std": 0.07409624010324478,
|
|
"rewards/env_reward/mean": 0.3232000172138214,
|
|
"rewards/env_reward/std": 0.1860000044107437,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 47
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 12.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 12.0,
|
|
"completions/max_terminated_length": 12.0,
|
|
"completions/mean_length": 10.5,
|
|
"completions/mean_terminated_length": 10.5,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.04,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 7.727506160736084,
|
|
"kl": 0.6681761667132378,
|
|
"learning_rate": 4.902777777777778e-05,
|
|
"loss": 0.0267,
|
|
"num_tokens": 112690.0,
|
|
"reward": 1.4063067436218262,
|
|
"reward_std": 0.5394194722175598,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.23232725262641907,
|
|
"rewards/belief_accuracy/std": 0.16305728256702423,
|
|
"rewards/env_reward/mean": 0.4395500123500824,
|
|
"rewards/env_reward/std": 0.4166736900806427,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 48
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.25,
|
|
"completions/mean_terminated_length": 9.25,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.04083333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.620209217071533,
|
|
"kl": 0.32542235124856234,
|
|
"learning_rate": 4.888888888888889e-05,
|
|
"loss": 0.013,
|
|
"num_tokens": 115127.0,
|
|
"reward": 0.45589596033096313,
|
|
"reward_std": 0.40303459763526917,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2432861626148224,
|
|
"rewards/belief_accuracy/std": 0.13326476514339447,
|
|
"rewards/env_reward/mean": -0.21597501635551453,
|
|
"rewards/env_reward/std": 0.0036500021815299988,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 49
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 28.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 28.0,
|
|
"completions/max_terminated_length": 28.0,
|
|
"completions/mean_length": 14.5,
|
|
"completions/mean_terminated_length": 14.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.041666666666666664,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 15.720799446105957,
|
|
"kl": 1.7742180861532688,
|
|
"learning_rate": 4.875e-05,
|
|
"loss": 0.071,
|
|
"num_tokens": 117585.0,
|
|
"reward": -0.6876518726348877,
|
|
"reward_std": 3.0439605712890625,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.14229519665241241,
|
|
"rewards/belief_accuracy/std": 0.23121023178100586,
|
|
"rewards/env_reward/mean": -0.743025004863739,
|
|
"rewards/env_reward/std": 1.5051672458648682,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 50
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0425,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.957500457763672,
|
|
"kl": 0.17545690201222897,
|
|
"learning_rate": 4.8611111111111115e-05,
|
|
"loss": 0.007,
|
|
"num_tokens": 119516.0,
|
|
"reward": 1.8904452323913574,
|
|
"reward_std": 0.34035390615463257,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3019692301750183,
|
|
"rewards/belief_accuracy/std": 0.07230889797210693,
|
|
"rewards/env_reward/mean": 0.6230250000953674,
|
|
"rewards/env_reward/std": 0.24345001578330994,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 51
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.043333333333333335,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.6418776512146,
|
|
"kl": 0.3459396343678236,
|
|
"learning_rate": 4.8472222222222224e-05,
|
|
"loss": 0.0138,
|
|
"num_tokens": 121954.0,
|
|
"reward": 0.47558823227882385,
|
|
"reward_std": 0.17540638148784637,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.24578773975372314,
|
|
"rewards/belief_accuracy/std": 0.12300436198711395,
|
|
"rewards/env_reward/mean": -0.20785000920295715,
|
|
"rewards/env_reward/std": 0.14000745117664337,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 52
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.04416666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.052071571350098,
|
|
"kl": 0.3293638424947858,
|
|
"learning_rate": 4.8333333333333334e-05,
|
|
"loss": 0.0132,
|
|
"num_tokens": 124393.0,
|
|
"reward": 1.111875057220459,
|
|
"reward_std": 0.417526513338089,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.34039586782455444,
|
|
"rewards/belief_accuracy/std": 0.07877691090106964,
|
|
"rewards/env_reward/mean": 0.027125000953674316,
|
|
"rewards/env_reward/std": 0.1343500018119812,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 53
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.045,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.564804553985596,
|
|
"kl": 0.12563097476959229,
|
|
"learning_rate": 4.819444444444445e-05,
|
|
"loss": 0.005,
|
|
"num_tokens": 126833.0,
|
|
"reward": -0.37219369411468506,
|
|
"reward_std": 3.258995294570923,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.1694604456424713,
|
|
"rewards/belief_accuracy/std": 0.25362879037857056,
|
|
"rewards/env_reward/mean": -0.5870499610900879,
|
|
"rewards/env_reward/std": 1.6088296175003052,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 54
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 8.25,
|
|
"completions/mean_terminated_length": 8.25,
|
|
"completions/min_length": 5.0,
|
|
"completions/min_terminated_length": 5.0,
|
|
"epoch": 0.04583333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 35.32516860961914,
|
|
"kl": 0.4959399476647377,
|
|
"learning_rate": 4.805555555555556e-05,
|
|
"loss": 0.0198,
|
|
"num_tokens": 129266.0,
|
|
"reward": 0.5137417316436768,
|
|
"reward_std": 1.1124905347824097,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.1392139196395874,
|
|
"rewards/belief_accuracy/std": 0.24298755824565887,
|
|
"rewards/env_reward/mean": 0.0349000059068203,
|
|
"rewards/env_reward/std": 0.3493712842464447,
|
|
"rewards/format_valid/mean": 0.875,
|
|
"rewards/format_valid/std": 0.25,
|
|
"step": 55
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.04666666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.5689568519592285,
|
|
"kl": 0.5865896865725517,
|
|
"learning_rate": 4.791666666666667e-05,
|
|
"loss": 0.0235,
|
|
"num_tokens": 131704.0,
|
|
"reward": 0.5574407577514648,
|
|
"reward_std": 0.4707197844982147,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.20113441348075867,
|
|
"rewards/belief_accuracy/std": 0.06771832704544067,
|
|
"rewards/env_reward/mean": -0.0639750063419342,
|
|
"rewards/env_reward/std": 0.37699612975120544,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 56
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0475,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.44246768951416,
|
|
"kl": 0.3799317330121994,
|
|
"learning_rate": 4.7777777777777784e-05,
|
|
"loss": 0.0152,
|
|
"num_tokens": 134142.0,
|
|
"reward": 0.5855016708374023,
|
|
"reward_std": 0.5364238023757935,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.24740056693553925,
|
|
"rewards/belief_accuracy/std": 0.19466152787208557,
|
|
"rewards/env_reward/mean": -0.13779999315738678,
|
|
"rewards/env_reward/std": 0.4736187756061554,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 57
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.04833333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.86322021484375,
|
|
"kl": 0.3641611896455288,
|
|
"learning_rate": 4.7638888888888887e-05,
|
|
"loss": 0.0146,
|
|
"num_tokens": 136072.0,
|
|
"reward": 1.1454858779907227,
|
|
"reward_std": 0.34598207473754883,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2346869856119156,
|
|
"rewards/belief_accuracy/std": 0.0894273892045021,
|
|
"rewards/env_reward/mean": 0.2609499990940094,
|
|
"rewards/env_reward/std": 0.06991129368543625,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 58
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.25,
|
|
"completions/mean_terminated_length": 9.666666984558105,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.049166666666666664,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 45.849605560302734,
|
|
"kl": 2.8015664890408516,
|
|
"learning_rate": 4.75e-05,
|
|
"loss": 0.1121,
|
|
"num_tokens": 138533.0,
|
|
"reward": 1.7629446983337402,
|
|
"reward_std": 0.3849683701992035,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.32485654950141907,
|
|
"rewards/belief_accuracy/std": 0.14582888782024384,
|
|
"rewards/env_reward/mean": 0.4922500252723694,
|
|
"rewards/env_reward/std": 0.1800999939441681,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 59
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 11.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 11.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 10.25,
|
|
"completions/mean_terminated_length": 10.25,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.05,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.147298336029053,
|
|
"kl": 0.15783802792429924,
|
|
"learning_rate": 4.736111111111111e-05,
|
|
"loss": 0.0063,
|
|
"num_tokens": 140974.0,
|
|
"reward": 1.0111374855041504,
|
|
"reward_std": 1.9640129804611206,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.22329160571098328,
|
|
"rewards/belief_accuracy/std": 0.2285340428352356,
|
|
"rewards/env_reward/mean": 0.19417500495910645,
|
|
"rewards/env_reward/std": 0.9323619604110718,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 60
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 10.75,
|
|
"completions/mean_terminated_length": 10.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.050833333333333335,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.107398509979248,
|
|
"kl": 0.6699433820322156,
|
|
"learning_rate": 4.722222222222222e-05,
|
|
"loss": 0.0268,
|
|
"num_tokens": 143417.0,
|
|
"reward": 0.7414954900741577,
|
|
"reward_std": 0.12289471924304962,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2869609594345093,
|
|
"rewards/belief_accuracy/std": 0.03287558630108833,
|
|
"rewards/env_reward/mean": -0.11292499303817749,
|
|
"rewards/env_reward/std": 0.02424999698996544,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 61
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.051666666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.9671472311019897,
|
|
"kl": 0.21522311307489872,
|
|
"learning_rate": 4.708333333333334e-05,
|
|
"loss": 0.0086,
|
|
"num_tokens": 145857.0,
|
|
"reward": 1.8655295372009277,
|
|
"reward_std": 0.22047176957130432,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.1922764927148819,
|
|
"rewards/belief_accuracy/std": 0.07349060475826263,
|
|
"rewards/env_reward/mean": 0.8258000016212463,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 62
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.0,
|
|
"completions/mean_terminated_length": 9.333333969116211,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0525,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 46.615821838378906,
|
|
"kl": 7.5339599046856165,
|
|
"learning_rate": 4.6944444444444446e-05,
|
|
"loss": 0.3014,
|
|
"num_tokens": 148317.0,
|
|
"reward": -0.0844120979309082,
|
|
"reward_std": 3.4652018547058105,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.16348761320114136,
|
|
"rewards/belief_accuracy/std": 0.24438007175922394,
|
|
"rewards/env_reward/mean": -0.38324999809265137,
|
|
"rewards/env_reward/std": 1.7579199075698853,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 63
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.05333333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.313786506652832,
|
|
"kl": 0.31290814094245434,
|
|
"learning_rate": 4.6805555555555556e-05,
|
|
"loss": 0.0125,
|
|
"num_tokens": 150756.0,
|
|
"reward": 1.4420485496520996,
|
|
"reward_std": 0.30519527196884155,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2965036630630493,
|
|
"rewards/belief_accuracy/std": 0.015631647780537605,
|
|
"rewards/env_reward/mean": 0.33502501249313354,
|
|
"rewards/env_reward/std": 0.1946970373392105,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 64
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.05416666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.699985980987549,
|
|
"kl": 0.27691997960209846,
|
|
"learning_rate": 4.666666666666667e-05,
|
|
"loss": 0.0111,
|
|
"num_tokens": 153195.0,
|
|
"reward": 0.6549558639526367,
|
|
"reward_std": 0.4356115162372589,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.09438945353031158,
|
|
"rewards/belief_accuracy/std": 0.07635381072759628,
|
|
"rewards/env_reward/mean": 0.2145249992609024,
|
|
"rewards/env_reward/std": 0.4134778380393982,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 65
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.055,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.48840594291687,
|
|
"kl": 0.23591649159789085,
|
|
"learning_rate": 4.652777777777778e-05,
|
|
"loss": 0.0094,
|
|
"num_tokens": 155634.0,
|
|
"reward": 1.1451208591461182,
|
|
"reward_std": 0.143072709441185,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2777777910232544,
|
|
"rewards/belief_accuracy/std": 0.06283816695213318,
|
|
"rewards/env_reward/mean": 0.17452499270439148,
|
|
"rewards/env_reward/std": 0.19420935213565826,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 66
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.05583333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.279496669769287,
|
|
"kl": 0.2593200672417879,
|
|
"learning_rate": 4.638888888888889e-05,
|
|
"loss": 0.0104,
|
|
"num_tokens": 158072.0,
|
|
"reward": 0.5202434062957764,
|
|
"reward_std": 0.3491542339324951,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.15712279081344604,
|
|
"rewards/belief_accuracy/std": 0.0946710854768753,
|
|
"rewards/env_reward/mean": -0.0007500015199184418,
|
|
"rewards/env_reward/std": 0.0485551580786705,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 67
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.056666666666666664,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.2196364402771,
|
|
"kl": 0.22027479112148285,
|
|
"learning_rate": 4.6250000000000006e-05,
|
|
"loss": 0.0088,
|
|
"num_tokens": 160511.0,
|
|
"reward": 1.1925969123840332,
|
|
"reward_std": 0.4943404197692871,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2660156190395355,
|
|
"rewards/belief_accuracy/std": 0.03546026349067688,
|
|
"rewards/env_reward/mean": 0.2296999990940094,
|
|
"rewards/env_reward/std": 0.3610000014305115,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 68
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0575,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.0931131839752197,
|
|
"kl": 0.31468209251761436,
|
|
"learning_rate": 4.6111111111111115e-05,
|
|
"loss": 0.0126,
|
|
"num_tokens": 162950.0,
|
|
"reward": 1.4288655519485474,
|
|
"reward_std": 0.21090544760227203,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2787468433380127,
|
|
"rewards/belief_accuracy/std": 0.07509276270866394,
|
|
"rewards/env_reward/mean": 0.3617500066757202,
|
|
"rewards/env_reward/std": 0.12350000441074371,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 69
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 17.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 17.0,
|
|
"completions/max_terminated_length": 17.0,
|
|
"completions/mean_length": 11.25,
|
|
"completions/mean_terminated_length": 11.25,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.058333333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.71396017074585,
|
|
"kl": 1.3337756432592869,
|
|
"learning_rate": 4.5972222222222225e-05,
|
|
"loss": 0.0534,
|
|
"num_tokens": 165395.0,
|
|
"reward": 1.2500306367874146,
|
|
"reward_std": 0.5057908892631531,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.30136024951934814,
|
|
"rewards/belief_accuracy/std": 0.09320782870054245,
|
|
"rewards/env_reward/mean": 0.1973000019788742,
|
|
"rewards/env_reward/std": 0.24144788086414337,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 70
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.059166666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.5546116828918457,
|
|
"kl": 0.4766240194439888,
|
|
"learning_rate": 4.5833333333333334e-05,
|
|
"loss": 0.0191,
|
|
"num_tokens": 167834.0,
|
|
"reward": 0.970757007598877,
|
|
"reward_std": 0.37021511793136597,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.24463149905204773,
|
|
"rewards/belief_accuracy/std": 0.12909035384655,
|
|
"rewards/env_reward/mean": 0.12457499653100967,
|
|
"rewards/env_reward/std": 0.02005000039935112,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 71
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 16.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 16.0,
|
|
"completions/max_terminated_length": 16.0,
|
|
"completions/mean_length": 11.5,
|
|
"completions/mean_terminated_length": 11.5,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.06,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.085524559020996,
|
|
"kl": 1.051513284444809,
|
|
"learning_rate": 4.569444444444444e-05,
|
|
"loss": 0.0421,
|
|
"num_tokens": 170280.0,
|
|
"reward": 1.3046754598617554,
|
|
"reward_std": 0.1669362485408783,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.191875159740448,
|
|
"rewards/belief_accuracy/std": 0.0556454099714756,
|
|
"rewards/env_reward/mean": 0.4526999890804291,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 72
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 15.5,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.060833333333333336,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 17.69377326965332,
|
|
"kl": 1.1864948254078627,
|
|
"learning_rate": 4.555555555555556e-05,
|
|
"loss": 0.0475,
|
|
"num_tokens": 172602.0,
|
|
"reward": -2.122437000274658,
|
|
"reward_std": 3.624743938446045,
|
|
"rewards/action_legal/mean": -0.5,
|
|
"rewards/action_legal/std": 0.5773502588272095,
|
|
"rewards/belief_accuracy/mean": 0.07886260747909546,
|
|
"rewards/belief_accuracy/std": 0.32305219769477844,
|
|
"rewards/env_reward/mean": -1.5393500328063965,
|
|
"rewards/env_reward/std": 1.6937329769134521,
|
|
"rewards/format_valid/mean": -0.5,
|
|
"rewards/format_valid/std": 1.7320507764816284,
|
|
"step": 73
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.06166666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.4644877910614014,
|
|
"kl": 0.1901670265942812,
|
|
"learning_rate": 4.541666666666667e-05,
|
|
"loss": 0.0076,
|
|
"num_tokens": 175042.0,
|
|
"reward": 1.3600037097930908,
|
|
"reward_std": 0.1827382892370224,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.24918042123317719,
|
|
"rewards/belief_accuracy/std": 0.07092051953077316,
|
|
"rewards/env_reward/mean": 0.3749750256538391,
|
|
"rewards/env_reward/std": 0.08854999393224716,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 74
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 12.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 12.0,
|
|
"completions/max_terminated_length": 12.0,
|
|
"completions/mean_length": 10.5,
|
|
"completions/mean_terminated_length": 10.5,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.0625,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.083766937255859,
|
|
"kl": 0.6269553881138563,
|
|
"learning_rate": 4.527777777777778e-05,
|
|
"loss": 0.0251,
|
|
"num_tokens": 177484.0,
|
|
"reward": -0.3148350715637207,
|
|
"reward_std": 3.2907352447509766,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.25676751136779785,
|
|
"rewards/belief_accuracy/std": 0.30544862151145935,
|
|
"rewards/env_reward/mean": -0.7234249711036682,
|
|
"rewards/env_reward/std": 1.5177685022354126,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 75
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.06333333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.858494997024536,
|
|
"kl": 1.4018143992871046,
|
|
"learning_rate": 4.5138888888888894e-05,
|
|
"loss": 0.0561,
|
|
"num_tokens": 179923.0,
|
|
"reward": 1.0085630416870117,
|
|
"reward_std": 0.5302945375442505,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.1935710310935974,
|
|
"rewards/belief_accuracy/std": 0.07880179584026337,
|
|
"rewards/env_reward/mean": 0.2519000172615051,
|
|
"rewards/env_reward/std": 0.2782000005245209,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 76
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.06416666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.091435432434082,
|
|
"kl": 0.4507467746734619,
|
|
"learning_rate": 4.5e-05,
|
|
"loss": 0.018,
|
|
"num_tokens": 182362.0,
|
|
"reward": 0.719524621963501,
|
|
"reward_std": 0.5247438549995422,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.32113736867904663,
|
|
"rewards/belief_accuracy/std": 0.06240236759185791,
|
|
"rewards/env_reward/mean": -0.19592499732971191,
|
|
"rewards/env_reward/std": 0.4110499918460846,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 77
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.065,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.242192268371582,
|
|
"kl": 0.6336696594953537,
|
|
"learning_rate": 4.486111111111111e-05,
|
|
"loss": 0.0253,
|
|
"num_tokens": 184802.0,
|
|
"reward": 1.1715035438537598,
|
|
"reward_std": 0.5318323373794556,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.31365954875946045,
|
|
"rewards/belief_accuracy/std": 0.16403073072433472,
|
|
"rewards/env_reward/mean": 0.12035000324249268,
|
|
"rewards/env_reward/std": 0.0738430991768837,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 78
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.06583333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.0246036052703857,
|
|
"kl": 0.21958505548536777,
|
|
"learning_rate": 4.472222222222223e-05,
|
|
"loss": 0.0088,
|
|
"num_tokens": 187242.0,
|
|
"reward": 0.20573002099990845,
|
|
"reward_std": 0.39767947793006897,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.18856000900268555,
|
|
"rewards/belief_accuracy/std": 0.034971851855516434,
|
|
"rewards/env_reward/mean": -0.2732999920845032,
|
|
"rewards/env_reward/std": 0.2833635210990906,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 79
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.06666666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1923185586929321,
|
|
"kl": 0.3896181844174862,
|
|
"learning_rate": 4.458333333333334e-05,
|
|
"loss": 0.0156,
|
|
"num_tokens": 189682.0,
|
|
"reward": 0.9259074926376343,
|
|
"reward_std": 0.12676748633384705,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.366219162940979,
|
|
"rewards/belief_accuracy/std": 0.04225583001971245,
|
|
"rewards/env_reward/mean": -0.148499995470047,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 80
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.0675,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.7216500639915466,
|
|
"kl": 0.3197862319648266,
|
|
"learning_rate": 4.4444444444444447e-05,
|
|
"loss": 0.0128,
|
|
"num_tokens": 192122.0,
|
|
"reward": 0.9888283014297485,
|
|
"reward_std": 0.10651447623968124,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2893427610397339,
|
|
"rewards/belief_accuracy/std": 0.03550482541322708,
|
|
"rewards/env_reward/mean": 0.047200001776218414,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 81
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.06833333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.201247453689575,
|
|
"kl": 0.8603321500122547,
|
|
"learning_rate": 4.4305555555555556e-05,
|
|
"loss": 0.0344,
|
|
"num_tokens": 194232.0,
|
|
"reward": 0.7670385241508484,
|
|
"reward_std": 0.4939156472682953,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.31966283917427063,
|
|
"rewards/belief_accuracy/std": 0.07682497054338455,
|
|
"rewards/env_reward/mean": -0.16130000352859497,
|
|
"rewards/env_reward/std": 0.2710742652416229,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 82
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.06916666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1376465559005737,
|
|
"kl": 0.2951663341373205,
|
|
"learning_rate": 4.4166666666666665e-05,
|
|
"loss": 0.0118,
|
|
"num_tokens": 196672.0,
|
|
"reward": 1.633320927619934,
|
|
"reward_std": 0.29003316164016724,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2325861155986786,
|
|
"rewards/belief_accuracy/std": 0.10147262364625931,
|
|
"rewards/env_reward/mean": 0.5903750061988831,
|
|
"rewards/env_reward/std": 0.039749979972839355,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 83
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 22.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 22.0,
|
|
"completions/max_terminated_length": 22.0,
|
|
"completions/mean_length": 13.0,
|
|
"completions/mean_terminated_length": 13.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.07,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 12.477323532104492,
|
|
"kl": 0.8052617497742176,
|
|
"learning_rate": 4.402777777777778e-05,
|
|
"loss": 0.0322,
|
|
"num_tokens": 199124.0,
|
|
"reward": 1.5077629089355469,
|
|
"reward_std": 0.10474622994661331,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.340520977973938,
|
|
"rewards/belief_accuracy/std": 0.03491540253162384,
|
|
"rewards/env_reward/mean": 0.290800005197525,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 84
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.07083333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.133931875228882,
|
|
"kl": 0.46640536189079285,
|
|
"learning_rate": 4.388888888888889e-05,
|
|
"loss": 0.0187,
|
|
"num_tokens": 201564.0,
|
|
"reward": 1.811617374420166,
|
|
"reward_std": 0.4033048748970032,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.30618077516555786,
|
|
"rewards/belief_accuracy/std": 0.11218413710594177,
|
|
"rewards/env_reward/mean": 0.5620499849319458,
|
|
"rewards/env_reward/std": 0.06709998846054077,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 85
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.07166666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.235334038734436,
|
|
"kl": 0.34326545894145966,
|
|
"learning_rate": 4.375e-05,
|
|
"loss": 0.0137,
|
|
"num_tokens": 204004.0,
|
|
"reward": 1.235045313835144,
|
|
"reward_std": 0.25660011172294617,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.17671510577201843,
|
|
"rewards/belief_accuracy/std": 0.08553336560726166,
|
|
"rewards/env_reward/mean": 0.436599999666214,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 86
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.0725,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.338106393814087,
|
|
"kl": 0.4051658548414707,
|
|
"learning_rate": 4.3611111111111116e-05,
|
|
"loss": 0.0162,
|
|
"num_tokens": 206303.0,
|
|
"reward": 1.0716049671173096,
|
|
"reward_std": 0.33843618631362915,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.28272244334220886,
|
|
"rewards/belief_accuracy/std": 0.06825492531061172,
|
|
"rewards/env_reward/mean": 0.11562499403953552,
|
|
"rewards/env_reward/std": 0.19865000247955322,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 87
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.5,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.07333333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 8.827055931091309,
|
|
"kl": 1.6250885128974915,
|
|
"learning_rate": 4.3472222222222225e-05,
|
|
"loss": 0.065,
|
|
"num_tokens": 208765.0,
|
|
"reward": 1.3299674987792969,
|
|
"reward_std": 0.1669912487268448,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3536558151245117,
|
|
"rewards/belief_accuracy/std": 0.055663757026195526,
|
|
"rewards/env_reward/mean": 0.1459999978542328,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 88
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.07416666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.412042260169983,
|
|
"kl": 0.633812952786684,
|
|
"learning_rate": 4.3333333333333334e-05,
|
|
"loss": 0.0254,
|
|
"num_tokens": 211205.0,
|
|
"reward": 1.214733600616455,
|
|
"reward_std": 0.11018647253513336,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.31789451837539673,
|
|
"rewards/belief_accuracy/std": 0.03672884404659271,
|
|
"rewards/env_reward/mean": 0.14069999754428864,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 89
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.075,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.7908711433410645,
|
|
"kl": 0.5902487859129906,
|
|
"learning_rate": 4.319444444444445e-05,
|
|
"loss": 0.0236,
|
|
"num_tokens": 213645.0,
|
|
"reward": 0.9393336176872253,
|
|
"reward_std": 0.3621884286403656,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.272844523191452,
|
|
"rewards/belief_accuracy/std": 0.1207294762134552,
|
|
"rewards/env_reward/mean": 0.047200001776218414,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 90
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 18.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 18.0,
|
|
"completions/max_terminated_length": 18.0,
|
|
"completions/mean_length": 12.0,
|
|
"completions/mean_terminated_length": 12.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.07583333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 8.704280853271484,
|
|
"kl": 1.1360323503613472,
|
|
"learning_rate": 4.305555555555556e-05,
|
|
"loss": 0.0454,
|
|
"num_tokens": 216093.0,
|
|
"reward": 1.6989027261734009,
|
|
"reward_std": 0.16673420369625092,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.20553426444530487,
|
|
"rewards/belief_accuracy/std": 0.05557806044816971,
|
|
"rewards/env_reward/mean": 0.6881999969482422,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 91
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.07666666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.743035793304443,
|
|
"kl": 0.7067995071411133,
|
|
"learning_rate": 4.291666666666667e-05,
|
|
"loss": 0.0283,
|
|
"num_tokens": 218204.0,
|
|
"reward": 1.5070440769195557,
|
|
"reward_std": 0.43784603476524353,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.24255633354187012,
|
|
"rewards/belief_accuracy/std": 0.10362043231725693,
|
|
"rewards/env_reward/mean": 0.48625001311302185,
|
|
"rewards/env_reward/std": 0.08950001001358032,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 92
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.0775,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3156030178070068,
|
|
"kl": 0.2781078703701496,
|
|
"learning_rate": 4.277777777777778e-05,
|
|
"loss": 0.0111,
|
|
"num_tokens": 220644.0,
|
|
"reward": 1.9765175580978394,
|
|
"reward_std": 0.13653874397277832,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.36422252655029297,
|
|
"rewards/belief_accuracy/std": 0.045512910932302475,
|
|
"rewards/env_reward/mean": 0.555899977684021,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 93
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.07833333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.8292455673217773,
|
|
"kl": 0.19214149564504623,
|
|
"learning_rate": 4.263888888888889e-05,
|
|
"loss": 0.0077,
|
|
"num_tokens": 223084.0,
|
|
"reward": 1.740675449371338,
|
|
"reward_std": 0.15911391377449036,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3569084405899048,
|
|
"rewards/belief_accuracy/std": 0.05303797498345375,
|
|
"rewards/env_reward/mean": 0.4133000075817108,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 94
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.07916666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.2383766174316406,
|
|
"kl": 0.572553887963295,
|
|
"learning_rate": 4.25e-05,
|
|
"loss": 0.0229,
|
|
"num_tokens": 225524.0,
|
|
"reward": 1.7529411315917969,
|
|
"reward_std": 0.2737933397293091,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.22559703886508942,
|
|
"rewards/belief_accuracy/std": 0.09126444160938263,
|
|
"rewards/env_reward/mean": 0.6840999722480774,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 95
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.08,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.971638798713684,
|
|
"kl": 0.5628382042050362,
|
|
"learning_rate": 4.236111111111111e-05,
|
|
"loss": 0.0225,
|
|
"num_tokens": 227963.0,
|
|
"reward": 1.681844711303711,
|
|
"reward_std": 0.03878229856491089,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.34314823150634766,
|
|
"rewards/belief_accuracy/std": 0.012927442789077759,
|
|
"rewards/env_reward/mean": 0.4016000032424927,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 96
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.08083333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.9865643978118896,
|
|
"kl": 0.22193955443799496,
|
|
"learning_rate": 4.222222222222222e-05,
|
|
"loss": 0.0089,
|
|
"num_tokens": 230403.0,
|
|
"reward": 1.3422417640686035,
|
|
"reward_std": 0.10156978666782379,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.33229726552963257,
|
|
"rewards/belief_accuracy/std": 0.03385661542415619,
|
|
"rewards/env_reward/mean": 0.19689999520778656,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 97
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.08166666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.4794961214065552,
|
|
"kl": 0.2080898880958557,
|
|
"learning_rate": 4.208333333333334e-05,
|
|
"loss": 0.0083,
|
|
"num_tokens": 232843.0,
|
|
"reward": -0.02455127239227295,
|
|
"reward_std": 3.4839961528778076,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.16232874989509583,
|
|
"rewards/belief_accuracy/std": 0.2421344518661499,
|
|
"rewards/env_reward/mean": -0.3410249948501587,
|
|
"rewards/env_reward/std": 1.7726500034332275,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 98
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.0825,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.3845081329345703,
|
|
"kl": 0.34126274287700653,
|
|
"learning_rate": 4.194444444444445e-05,
|
|
"loss": 0.0137,
|
|
"num_tokens": 235283.0,
|
|
"reward": 1.6563684940338135,
|
|
"reward_std": 0.09499318897724152,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.337656170129776,
|
|
"rewards/belief_accuracy/std": 0.031664397567510605,
|
|
"rewards/env_reward/mean": 0.39559999108314514,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 99
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.0,
|
|
"completions/mean_terminated_length": 9.333333969116211,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.08333333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.240288257598877,
|
|
"kl": 0.7164427638053894,
|
|
"learning_rate": 4.1805555555555556e-05,
|
|
"loss": 0.0287,
|
|
"num_tokens": 237235.0,
|
|
"reward": 1.2336010932922363,
|
|
"reward_std": 0.6653106212615967,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.18393371999263763,
|
|
"rewards/belief_accuracy/std": 0.15957118570804596,
|
|
"rewards/env_reward/mean": 0.4211999773979187,
|
|
"rewards/env_reward/std": 0.41892534494400024,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.08416666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.361682176589966,
|
|
"kl": 0.2631294522434473,
|
|
"learning_rate": 4.166666666666667e-05,
|
|
"loss": 0.0105,
|
|
"num_tokens": 239675.0,
|
|
"reward": 1.886284351348877,
|
|
"reward_std": 0.20986472070217133,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.24569477140903473,
|
|
"rewards/belief_accuracy/std": 0.06995491683483124,
|
|
"rewards/env_reward/mean": 0.7328000068664551,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 101
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.085,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.8391385078430176,
|
|
"kl": 0.24248860776424408,
|
|
"learning_rate": 4.152777777777778e-05,
|
|
"loss": 0.0097,
|
|
"num_tokens": 241607.0,
|
|
"reward": 1.4985287189483643,
|
|
"reward_std": 0.12530909478664398,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3252928853034973,
|
|
"rewards/belief_accuracy/std": 0.041769690811634064,
|
|
"rewards/env_reward/mean": 0.3151000142097473,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 102
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.5,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.08583333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 21.890640258789062,
|
|
"kl": 0.5941529143601656,
|
|
"learning_rate": 4.138888888888889e-05,
|
|
"loss": 0.0238,
|
|
"num_tokens": 244069.0,
|
|
"reward": 0.44205623865127563,
|
|
"reward_std": 0.18975834548473358,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.22688540816307068,
|
|
"rewards/belief_accuracy/std": 0.06325279176235199,
|
|
"rewards/env_reward/mean": -0.1923999935388565,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 103
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.08666666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.787078380584717,
|
|
"kl": 0.2444281131029129,
|
|
"learning_rate": 4.125e-05,
|
|
"loss": 0.0098,
|
|
"num_tokens": 246509.0,
|
|
"reward": 1.6253175735473633,
|
|
"reward_std": 0.14385031163692474,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3388558626174927,
|
|
"rewards/belief_accuracy/std": 0.04795009270310402,
|
|
"rewards/env_reward/mean": 0.3725000023841858,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 104
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.0875,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.7878473997116089,
|
|
"kl": 0.2907773107290268,
|
|
"learning_rate": 4.111111111111111e-05,
|
|
"loss": 0.0116,
|
|
"num_tokens": 248949.0,
|
|
"reward": 1.292004108428955,
|
|
"reward_std": 0.19697506725788116,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.26045137643814087,
|
|
"rewards/belief_accuracy/std": 0.06565836817026138,
|
|
"rewards/env_reward/mean": 0.30709999799728394,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.08833333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 0.9713439345359802,
|
|
"kl": 0.5633938759565353,
|
|
"learning_rate": 4.0972222222222225e-05,
|
|
"loss": 0.0225,
|
|
"num_tokens": 251389.0,
|
|
"reward": 1.2565593719482422,
|
|
"reward_std": 0.05555558204650879,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.37923645973205566,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.045899998396635056,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 106
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.08916666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3064583539962769,
|
|
"kl": 0.3265066295862198,
|
|
"learning_rate": 4.0833333333333334e-05,
|
|
"loss": 0.0131,
|
|
"num_tokens": 253829.0,
|
|
"reward": 1.9252145290374756,
|
|
"reward_std": 0.06414999067783356,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2697714567184448,
|
|
"rewards/belief_accuracy/std": 0.02138334885239601,
|
|
"rewards/env_reward/mean": 0.7106000185012817,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 107
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.09,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.1808035373687744,
|
|
"kl": 0.33351393789052963,
|
|
"learning_rate": 4.0694444444444444e-05,
|
|
"loss": 0.0133,
|
|
"num_tokens": 256269.0,
|
|
"reward": 0.6878499984741211,
|
|
"reward_std": 0.24345053732395172,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.1825166642665863,
|
|
"rewards/belief_accuracy/std": 0.08115018159151077,
|
|
"rewards/env_reward/mean": 0.06019999831914902,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 108
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.09083333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2036610841751099,
|
|
"kl": 0.23895448073744774,
|
|
"learning_rate": 4.055555555555556e-05,
|
|
"loss": 0.0096,
|
|
"num_tokens": 258709.0,
|
|
"reward": 1.0095196962356567,
|
|
"reward_std": 0.07835428416728973,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.28903988003730774,
|
|
"rewards/belief_accuracy/std": 0.026118090376257896,
|
|
"rewards/env_reward/mean": 0.06159999966621399,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 109
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.75,
|
|
"completions/mean_terminated_length": 9.75,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.09166666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.1538102626800537,
|
|
"kl": 0.4214295297861099,
|
|
"learning_rate": 4.041666666666667e-05,
|
|
"loss": 0.0169,
|
|
"num_tokens": 260824.0,
|
|
"reward": 1.422867774963379,
|
|
"reward_std": 0.5747905373573303,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3610350489616394,
|
|
"rewards/belief_accuracy/std": 0.08047888427972794,
|
|
"rewards/env_reward/mean": 0.19317498803138733,
|
|
"rewards/env_reward/std": 0.26084083318710327,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.5,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.0925,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.361459255218506,
|
|
"kl": 0.5647962130606174,
|
|
"learning_rate": 4.027777777777778e-05,
|
|
"loss": 0.0226,
|
|
"num_tokens": 263286.0,
|
|
"reward": 0.67694091796875,
|
|
"reward_std": 0.2001858800649643,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.30193030834198,
|
|
"rewards/belief_accuracy/std": 0.0667286217212677,
|
|
"rewards/env_reward/mean": -0.1859000027179718,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 111
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.25,
|
|
"completions/mean_terminated_length": 9.25,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.09333333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.111879348754883,
|
|
"kl": 0.48548348993062973,
|
|
"learning_rate": 4.0138888888888894e-05,
|
|
"loss": 0.0194,
|
|
"num_tokens": 265395.0,
|
|
"reward": 1.0738086700439453,
|
|
"reward_std": 0.5100609660148621,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.32505708932876587,
|
|
"rewards/belief_accuracy/std": 0.06295914947986603,
|
|
"rewards/env_reward/mean": 0.03242500126361847,
|
|
"rewards/env_reward/std": 0.22145001590251923,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 112
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.09416666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.7761991024017334,
|
|
"kl": 0.44813764840364456,
|
|
"learning_rate": 4e-05,
|
|
"loss": 0.0179,
|
|
"num_tokens": 267835.0,
|
|
"reward": 1.1807773113250732,
|
|
"reward_std": 0.09589369595050812,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.36737579107284546,
|
|
"rewards/belief_accuracy/std": 0.031964562833309174,
|
|
"rewards/env_reward/mean": 0.019099999219179153,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 113
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.095,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.152681827545166,
|
|
"kl": 0.19383717328310013,
|
|
"learning_rate": 3.986111111111111e-05,
|
|
"loss": 0.0078,
|
|
"num_tokens": 270275.0,
|
|
"reward": 1.1284863948822021,
|
|
"reward_std": 0.14397799968719482,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.34184545278549194,
|
|
"rewards/belief_accuracy/std": 0.04799266904592514,
|
|
"rewards/env_reward/mean": 0.03530000150203705,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 114
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.09583333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.2871294021606445,
|
|
"kl": 0.35344624519348145,
|
|
"learning_rate": 3.972222222222222e-05,
|
|
"loss": 0.0141,
|
|
"num_tokens": 272715.0,
|
|
"reward": 2.019130229949951,
|
|
"reward_std": 0.05555558204650879,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2832767069339752,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.7462000250816345,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 115
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.09666666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.147212982177734,
|
|
"kl": 0.33963513001799583,
|
|
"learning_rate": 3.958333333333333e-05,
|
|
"loss": 0.0136,
|
|
"num_tokens": 274441.0,
|
|
"reward": 0.8702385425567627,
|
|
"reward_std": 0.22163395583629608,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.33136284351348877,
|
|
"rewards/belief_accuracy/std": 0.05834528058767319,
|
|
"rewards/env_reward/mean": -0.11590000241994858,
|
|
"rewards/env_reward/std": 0.22251078486442566,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 116
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.5,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.0975,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.5441172122955322,
|
|
"kl": 0.2427344862371683,
|
|
"learning_rate": 3.944444444444445e-05,
|
|
"loss": 0.0097,
|
|
"num_tokens": 276903.0,
|
|
"reward": 1.56264066696167,
|
|
"reward_std": 0.22222228348255157,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3731135427951813,
|
|
"rewards/belief_accuracy/std": 0.07407407462596893,
|
|
"rewards/env_reward/mean": 0.2621999979019165,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 117
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 9.5,
|
|
"completions/mean_terminated_length": 9.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.09833333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.864804267883301,
|
|
"kl": 0.4837944805622101,
|
|
"learning_rate": 3.9305555555555556e-05,
|
|
"loss": 0.0194,
|
|
"num_tokens": 279013.0,
|
|
"reward": 1.7349917888641357,
|
|
"reward_std": 0.3733745217323303,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3373388946056366,
|
|
"rewards/belief_accuracy/std": 0.03984580561518669,
|
|
"rewards/env_reward/mean": 0.4486500024795532,
|
|
"rewards/env_reward/std": 0.24901118874549866,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 118
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.09916666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.8735086917877197,
|
|
"kl": 0.2612016424536705,
|
|
"learning_rate": 3.9166666666666665e-05,
|
|
"loss": 0.0104,
|
|
"num_tokens": 281453.0,
|
|
"reward": 1.2192261219024658,
|
|
"reward_std": 0.18506474792957306,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2666420042514801,
|
|
"rewards/belief_accuracy/std": 0.061688266694545746,
|
|
"rewards/env_reward/mean": 0.24619999527931213,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 119
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.73070228099823,
|
|
"kl": 0.5154507085680962,
|
|
"learning_rate": 3.902777777777778e-05,
|
|
"loss": 0.0206,
|
|
"num_tokens": 283893.0,
|
|
"reward": 1.4962975978851318,
|
|
"reward_std": 0.13757789134979248,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.31319916248321533,
|
|
"rewards/belief_accuracy/std": 0.045859288424253464,
|
|
"rewards/env_reward/mean": 0.337799996137619,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 16.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 16.0,
|
|
"completions/max_terminated_length": 16.0,
|
|
"completions/mean_length": 11.5,
|
|
"completions/mean_terminated_length": 11.5,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.10083333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.9448511600494385,
|
|
"kl": 0.602842066437006,
|
|
"learning_rate": 3.888888888888889e-05,
|
|
"loss": 0.0241,
|
|
"num_tokens": 286339.0,
|
|
"reward": 1.649629831314087,
|
|
"reward_std": 0.10207336395978928,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.4207599461078644,
|
|
"rewards/belief_accuracy/std": 0.0340244434773922,
|
|
"rewards/env_reward/mean": 0.2249000072479248,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 121
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.10166666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.713958501815796,
|
|
"kl": 0.38420072570443153,
|
|
"learning_rate": 3.875e-05,
|
|
"loss": 0.0154,
|
|
"num_tokens": 288779.0,
|
|
"reward": 1.3020188808441162,
|
|
"reward_std": 0.17013852298259735,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.20623962581157684,
|
|
"rewards/belief_accuracy/std": 0.05671283230185509,
|
|
"rewards/env_reward/mean": 0.4221999943256378,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 122
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1025,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.8310298919677734,
|
|
"kl": 0.19116491079330444,
|
|
"learning_rate": 3.8611111111111116e-05,
|
|
"loss": 0.0076,
|
|
"num_tokens": 291219.0,
|
|
"reward": 0.7856130599975586,
|
|
"reward_std": 0.1111110970377922,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3014543652534485,
|
|
"rewards/belief_accuracy/std": 0.03703703731298447,
|
|
"rewards/env_reward/mean": -0.11249999701976776,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 123
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 16.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 16.0,
|
|
"completions/max_terminated_length": 16.0,
|
|
"completions/mean_length": 11.5,
|
|
"completions/mean_terminated_length": 11.5,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.10333333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.588038444519043,
|
|
"kl": 0.5185928121209145,
|
|
"learning_rate": 3.8472222222222225e-05,
|
|
"loss": 0.0207,
|
|
"num_tokens": 293665.0,
|
|
"reward": 1.210307240486145,
|
|
"reward_std": 0.10638084262609482,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.22401908040046692,
|
|
"rewards/belief_accuracy/std": 0.03546026349067688,
|
|
"rewards/env_reward/mean": 0.3255000114440918,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 124
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 11.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 11.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 10.25,
|
|
"completions/mean_terminated_length": 10.25,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.10416666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.076918363571167,
|
|
"kl": 0.277778223156929,
|
|
"learning_rate": 3.8333333333333334e-05,
|
|
"loss": 0.0111,
|
|
"num_tokens": 296106.0,
|
|
"reward": 1.3835383653640747,
|
|
"reward_std": 0.09814538806676865,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3789128065109253,
|
|
"rewards/belief_accuracy/std": 0.03271512687206268,
|
|
"rewards/env_reward/mean": 0.13120000064373016,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 125
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.105,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.4967398643493652,
|
|
"kl": 0.3801155686378479,
|
|
"learning_rate": 3.8194444444444444e-05,
|
|
"loss": 0.0152,
|
|
"num_tokens": 298546.0,
|
|
"reward": 1.9718296527862549,
|
|
"reward_std": 0.14045073091983795,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.35695987939834595,
|
|
"rewards/belief_accuracy/std": 0.04681692272424698,
|
|
"rewards/env_reward/mean": 0.567300021648407,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 126
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.10583333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.761016368865967,
|
|
"kl": 0.32585281878709793,
|
|
"learning_rate": 3.805555555555555e-05,
|
|
"loss": 0.013,
|
|
"num_tokens": 300658.0,
|
|
"reward": 2.205615282058716,
|
|
"reward_std": 0.4304605722427368,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.339000940322876,
|
|
"rewards/belief_accuracy/std": 0.02138334885239601,
|
|
"rewards/env_reward/mean": 0.7590750455856323,
|
|
"rewards/env_reward/std": 0.2601499855518341,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 127
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 11.0,
|
|
"completions/mean_terminated_length": 11.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.10666666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.960784435272217,
|
|
"kl": 1.1858660280704498,
|
|
"learning_rate": 3.791666666666667e-05,
|
|
"loss": 0.0474,
|
|
"num_tokens": 303102.0,
|
|
"reward": 1.9060065746307373,
|
|
"reward_std": 0.24637286365032196,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3400188684463501,
|
|
"rewards/belief_accuracy/std": 0.08212429285049438,
|
|
"rewards/env_reward/mean": 0.5572999715805054,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 128
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 12.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 12.0,
|
|
"completions/max_terminated_length": 12.0,
|
|
"completions/mean_length": 10.5,
|
|
"completions/mean_terminated_length": 10.5,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1075,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.856426954269409,
|
|
"kl": 0.44381893426179886,
|
|
"learning_rate": 3.777777777777778e-05,
|
|
"loss": 0.0178,
|
|
"num_tokens": 305220.0,
|
|
"reward": 1.8646245002746582,
|
|
"reward_std": 0.11111116409301758,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.18142487108707428,
|
|
"rewards/belief_accuracy/std": 0.03703703731298447,
|
|
"rewards/env_reward/mean": 0.8468999862670898,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 129
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.10833333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.8265440464019775,
|
|
"kl": 0.38515937700867653,
|
|
"learning_rate": 3.763888888888889e-05,
|
|
"loss": 0.0154,
|
|
"num_tokens": 307152.0,
|
|
"reward": 1.3294965028762817,
|
|
"reward_std": 0.09841816127300262,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.36771130561828613,
|
|
"rewards/belief_accuracy/std": 0.07701562345027924,
|
|
"rewards/env_reward/mean": 0.11757499724626541,
|
|
"rewards/env_reward/std": 0.18415001034736633,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.10916666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.664041519165039,
|
|
"kl": 0.21777665056288242,
|
|
"learning_rate": 3.7500000000000003e-05,
|
|
"loss": 0.0087,
|
|
"num_tokens": 309592.0,
|
|
"reward": 1.1950082778930664,
|
|
"reward_std": 0.15713481605052948,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3575194478034973,
|
|
"rewards/belief_accuracy/std": 0.052378278225660324,
|
|
"rewards/env_reward/mean": 0.04830000177025795,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 131
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 13.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 13.0,
|
|
"completions/max_terminated_length": 13.0,
|
|
"completions/mean_length": 10.75,
|
|
"completions/mean_terminated_length": 10.75,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.11,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.597440481185913,
|
|
"kl": 1.7791741862893105,
|
|
"learning_rate": 3.736111111111111e-05,
|
|
"loss": 0.0712,
|
|
"num_tokens": 312035.0,
|
|
"reward": 2.0074949264526367,
|
|
"reward_std": 0.09213761240243912,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.4375982880592346,
|
|
"rewards/belief_accuracy/std": 0.030712537467479706,
|
|
"rewards/env_reward/mean": 0.42980000376701355,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 132
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.11083333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.8326061964035034,
|
|
"kl": 0.35671964287757874,
|
|
"learning_rate": 3.722222222222222e-05,
|
|
"loss": 0.0143,
|
|
"num_tokens": 314335.0,
|
|
"reward": 1.8839524984359741,
|
|
"reward_std": 0.048975855112075806,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.43421751260757446,
|
|
"rewards/belief_accuracy/std": 0.016325272619724274,
|
|
"rewards/env_reward/mean": 0.35420000553131104,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 133
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.11166666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.641361951828003,
|
|
"kl": 0.3387632220983505,
|
|
"learning_rate": 3.708333333333334e-05,
|
|
"loss": 0.0136,
|
|
"num_tokens": 316447.0,
|
|
"reward": 2.0452380180358887,
|
|
"reward_std": 0.317529559135437,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.443654328584671,
|
|
"rewards/belief_accuracy/std": 0.021532177925109863,
|
|
"rewards/env_reward/mean": 0.4428499937057495,
|
|
"rewards/env_reward/std": 0.1823849380016327,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 134
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1125,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 23.68372917175293,
|
|
"kl": 9.03631467744708,
|
|
"learning_rate": 3.694444444444445e-05,
|
|
"loss": 0.3615,
|
|
"num_tokens": 318887.0,
|
|
"reward": 1.3748897314071655,
|
|
"reward_std": 0.185723215341568,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.35687991976737976,
|
|
"rewards/belief_accuracy/std": 0.06190773472189903,
|
|
"rewards/env_reward/mean": 0.16949999332427979,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 135
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.11333333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.9297913312911987,
|
|
"kl": 0.3083711676299572,
|
|
"learning_rate": 3.6805555555555556e-05,
|
|
"loss": 0.0123,
|
|
"num_tokens": 321327.0,
|
|
"reward": 2.360555648803711,
|
|
"reward_std": 0.05555558204650879,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.25393518805503845,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 1.0325000286102295,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 136
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 27.0,
|
|
"completions/mean_length": 19.75,
|
|
"completions/mean_terminated_length": 15.666666984558105,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.11416666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.193662166595459,
|
|
"kl": 0.33080876618623734,
|
|
"learning_rate": 3.6666666666666666e-05,
|
|
"loss": 0.0132,
|
|
"num_tokens": 323806.0,
|
|
"reward": 1.2503528594970703,
|
|
"reward_std": 0.10516492277383804,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3806675672531128,
|
|
"rewards/belief_accuracy/std": 0.03505498543381691,
|
|
"rewards/env_reward/mean": 0.03889999911189079,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 137
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.115,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.0646451711654663,
|
|
"kl": 0.4817114397883415,
|
|
"learning_rate": 3.6527777777777775e-05,
|
|
"loss": 0.0193,
|
|
"num_tokens": 326246.0,
|
|
"reward": 1.6292959451675415,
|
|
"reward_std": 0.10631169378757477,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.42438197135925293,
|
|
"rewards/belief_accuracy/std": 0.03543723374605179,
|
|
"rewards/env_reward/mean": 0.20409999787807465,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 138
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.11583333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.5362420082092285,
|
|
"kl": 0.3246644027531147,
|
|
"learning_rate": 3.638888888888889e-05,
|
|
"loss": 0.013,
|
|
"num_tokens": 328686.0,
|
|
"reward": 1.1507596969604492,
|
|
"reward_std": 0.09127989411354065,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3048698902130127,
|
|
"rewards/belief_accuracy/std": 0.028253890573978424,
|
|
"rewards/env_reward/mean": 0.12409999966621399,
|
|
"rewards/env_reward/std": 0.02619999647140503,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 139
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.11666666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.2923882007598877,
|
|
"kl": 0.3384244814515114,
|
|
"learning_rate": 3.625e-05,
|
|
"loss": 0.0135,
|
|
"num_tokens": 330414.0,
|
|
"reward": 1.2623817920684814,
|
|
"reward_std": 0.3048122227191925,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.28640225529670715,
|
|
"rewards/belief_accuracy/std": 0.01621234230697155,
|
|
"rewards/env_reward/mean": 0.23544999957084656,
|
|
"rewards/env_reward/std": 0.234499990940094,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1175,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.7288119792938232,
|
|
"kl": 0.36574723571538925,
|
|
"learning_rate": 3.611111111111111e-05,
|
|
"loss": 0.0146,
|
|
"num_tokens": 332854.0,
|
|
"reward": 1.373946189880371,
|
|
"reward_std": 0.09072189033031464,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.27031537890434265,
|
|
"rewards/belief_accuracy/std": 0.03024062141776085,
|
|
"rewards/env_reward/mean": 0.34200000762939453,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 141
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.11833333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.0891668796539307,
|
|
"kl": 0.3505154550075531,
|
|
"learning_rate": 3.5972222222222225e-05,
|
|
"loss": 0.014,
|
|
"num_tokens": 334970.0,
|
|
"reward": 0.7695170640945435,
|
|
"reward_std": 0.18945620954036713,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2285890281200409,
|
|
"rewards/belief_accuracy/std": 0.06315205991268158,
|
|
"rewards/env_reward/mean": 0.02250000089406967,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 142
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.11916666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.0063207149505615,
|
|
"kl": 0.2529054693877697,
|
|
"learning_rate": 3.5833333333333335e-05,
|
|
"loss": 0.0101,
|
|
"num_tokens": 337410.0,
|
|
"reward": 0.9304206371307373,
|
|
"reward_std": 0.055555541068315506,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.35887354612350464,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": -0.13079999387264252,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 143
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.12,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.956833839416504,
|
|
"kl": 0.5815155953168869,
|
|
"learning_rate": 3.5694444444444444e-05,
|
|
"loss": 0.0233,
|
|
"num_tokens": 339850.0,
|
|
"reward": 1.0631434917449951,
|
|
"reward_std": 0.166666641831398,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.1861644983291626,
|
|
"rewards/belief_accuracy/std": 0.0555555522441864,
|
|
"rewards/env_reward/mean": 0.30309998989105225,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 144
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 15.75,
|
|
"completions/mean_terminated_length": 10.333333969116211,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.12083333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.329380035400391,
|
|
"kl": 0.8242634683847427,
|
|
"learning_rate": 3.555555555555556e-05,
|
|
"loss": 0.033,
|
|
"num_tokens": 342313.0,
|
|
"reward": 1.0987353324890137,
|
|
"reward_std": 0.14344383776187897,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3755284249782562,
|
|
"rewards/belief_accuracy/std": 0.04781460389494896,
|
|
"rewards/env_reward/mean": -0.051899999380111694,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 145
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.12166666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.634685754776001,
|
|
"kl": 0.39855731278657913,
|
|
"learning_rate": 3.541666666666667e-05,
|
|
"loss": 0.0159,
|
|
"num_tokens": 344753.0,
|
|
"reward": 1.4183378219604492,
|
|
"reward_std": 0.06828323751688004,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.31596264243125916,
|
|
"rewards/belief_accuracy/std": 0.02276109904050827,
|
|
"rewards/env_reward/mean": 0.28029999136924744,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 146
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1225,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.3948684930801392,
|
|
"kl": 0.38383544236421585,
|
|
"learning_rate": 3.527777777777778e-05,
|
|
"loss": 0.0154,
|
|
"num_tokens": 347193.0,
|
|
"reward": 1.1515657901763916,
|
|
"reward_std": 0.06009494140744209,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2850886285305023,
|
|
"rewards/belief_accuracy/std": 0.020031645894050598,
|
|
"rewards/env_reward/mean": 0.16419999301433563,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 147
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.12333333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.6996270418167114,
|
|
"kl": 0.6058650612831116,
|
|
"learning_rate": 3.513888888888889e-05,
|
|
"loss": 0.0242,
|
|
"num_tokens": 349633.0,
|
|
"reward": 1.1596319675445557,
|
|
"reward_std": 0.10638084262609482,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.30072730779647827,
|
|
"rewards/belief_accuracy/std": 0.03546027094125748,
|
|
"rewards/env_reward/mean": 0.13830000162124634,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 148
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.12416666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.426046848297119,
|
|
"kl": 0.34162479639053345,
|
|
"learning_rate": 3.5e-05,
|
|
"loss": 0.0137,
|
|
"num_tokens": 352073.0,
|
|
"reward": 1.1792874336242676,
|
|
"reward_std": 0.09853797405958176,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.31412917375564575,
|
|
"rewards/belief_accuracy/std": 0.03284599259495735,
|
|
"rewards/env_reward/mean": 0.12460000067949295,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 149
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 19.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 19.0,
|
|
"completions/max_terminated_length": 19.0,
|
|
"completions/mean_length": 12.25,
|
|
"completions/mean_terminated_length": 12.25,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.125,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 7.012240409851074,
|
|
"kl": 0.6527384743094444,
|
|
"learning_rate": 3.486111111111111e-05,
|
|
"loss": 0.0261,
|
|
"num_tokens": 354522.0,
|
|
"reward": 0.9352785348892212,
|
|
"reward_std": 0.07883862406015396,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.26061785221099854,
|
|
"rewards/belief_accuracy/std": 0.030240608379244804,
|
|
"rewards/env_reward/mean": 0.06894999742507935,
|
|
"rewards/env_reward/std": 0.010099999606609344,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 11.0,
|
|
"completions/mean_terminated_length": 11.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.12583333333333332,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 12.943872451782227,
|
|
"kl": 0.6852549761533737,
|
|
"learning_rate": 3.472222222222222e-05,
|
|
"loss": 0.0274,
|
|
"num_tokens": 356966.0,
|
|
"reward": 1.8197388648986816,
|
|
"reward_std": 0.12830005586147308,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.28171297907829285,
|
|
"rewards/belief_accuracy/std": 0.04276669770479202,
|
|
"rewards/env_reward/mean": 0.6164000034332275,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 151
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 11.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 11.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 10.25,
|
|
"completions/mean_terminated_length": 10.25,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.12666666666666668,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.472937822341919,
|
|
"kl": 0.3669867143034935,
|
|
"learning_rate": 3.458333333333333e-05,
|
|
"loss": 0.0147,
|
|
"num_tokens": 359407.0,
|
|
"reward": 1.2027755975723267,
|
|
"reward_std": 0.02803901769220829,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3199460506439209,
|
|
"rewards/belief_accuracy/std": 0.009259253740310669,
|
|
"rewards/env_reward/mean": 0.12862500548362732,
|
|
"rewards/env_reward/std": 0.012850001454353333,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 152
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1275,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.7880311012268066,
|
|
"kl": 0.8225667700171471,
|
|
"learning_rate": 3.444444444444445e-05,
|
|
"loss": 0.0329,
|
|
"num_tokens": 361519.0,
|
|
"reward": 2.0395917892456055,
|
|
"reward_std": 0.3787125051021576,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3322972357273102,
|
|
"rewards/belief_accuracy/std": 0.03908432275056839,
|
|
"rewards/env_reward/mean": 0.6618000268936157,
|
|
"rewards/env_reward/std": 0.20403560996055603,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 153
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.12833333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.15445613861084,
|
|
"kl": 0.41129303723573685,
|
|
"learning_rate": 3.430555555555556e-05,
|
|
"loss": 0.0165,
|
|
"num_tokens": 363959.0,
|
|
"reward": 0.5192468166351318,
|
|
"reward_std": 0.08724445104598999,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.17216560244560242,
|
|
"rewards/belief_accuracy/std": 0.018518514931201935,
|
|
"rewards/env_reward/mean": -0.03149999678134918,
|
|
"rewards/env_reward/std": 0.09520000219345093,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 154
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.12916666666666668,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.1737233400344849,
|
|
"kl": 0.35325073450803757,
|
|
"learning_rate": 3.4166666666666666e-05,
|
|
"loss": 0.0141,
|
|
"num_tokens": 366399.0,
|
|
"reward": 0.9217313528060913,
|
|
"reward_std": 0.055555522441864014,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2698771059513092,
|
|
"rewards/belief_accuracy/std": 0.018518507480621338,
|
|
"rewards/env_reward/mean": 0.0414000004529953,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 13.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 13.0,
|
|
"completions/max_terminated_length": 13.0,
|
|
"completions/mean_length": 10.75,
|
|
"completions/mean_terminated_length": 10.75,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.13,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 11.461524963378906,
|
|
"kl": 0.7672573626041412,
|
|
"learning_rate": 3.402777777777778e-05,
|
|
"loss": 0.0307,
|
|
"num_tokens": 368842.0,
|
|
"reward": 1.8642927408218384,
|
|
"reward_std": 0.12469932436943054,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.40906423330307007,
|
|
"rewards/belief_accuracy/std": 0.04156642034649849,
|
|
"rewards/env_reward/mean": 0.391400009393692,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 156
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.13083333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.6521883010864258,
|
|
"kl": 0.3942847102880478,
|
|
"learning_rate": 3.388888888888889e-05,
|
|
"loss": 0.0158,
|
|
"num_tokens": 371282.0,
|
|
"reward": 1.6033741235733032,
|
|
"reward_std": 0.05555558204650879,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.37149137258529663,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.29260000586509705,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 157
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.13166666666666665,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.972644329071045,
|
|
"kl": 0.40470410883426666,
|
|
"learning_rate": 3.375000000000001e-05,
|
|
"loss": 0.0162,
|
|
"num_tokens": 373722.0,
|
|
"reward": 0.5326824188232422,
|
|
"reward_std": 0.02388697862625122,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.29159414768218994,
|
|
"rewards/belief_accuracy/std": 0.007962316274642944,
|
|
"rewards/env_reward/mean": -0.2614000141620636,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 158
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1325,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.185356855392456,
|
|
"kl": 0.3720128685235977,
|
|
"learning_rate": 3.3611111111111116e-05,
|
|
"loss": 0.0149,
|
|
"num_tokens": 376162.0,
|
|
"reward": 1.7469711303710938,
|
|
"reward_std": 0.039430540055036545,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3106195032596588,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.5100749731063843,
|
|
"rewards/env_reward/std": 0.01074999663978815,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 159
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 15.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 15.0,
|
|
"completions/max_terminated_length": 15.0,
|
|
"completions/mean_length": 11.25,
|
|
"completions/mean_terminated_length": 11.25,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.13333333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.548361301422119,
|
|
"kl": 0.527301162481308,
|
|
"learning_rate": 3.347222222222222e-05,
|
|
"loss": 0.0211,
|
|
"num_tokens": 378607.0,
|
|
"reward": 1.302304744720459,
|
|
"reward_std": 0.1430416703224182,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3570098876953125,
|
|
"rewards/belief_accuracy/std": 0.030240608379244804,
|
|
"rewards/env_reward/mean": 0.12084999680519104,
|
|
"rewards/env_reward/std": 0.07372763007879257,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.13416666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.99193000793457,
|
|
"kl": 0.3444783613085747,
|
|
"learning_rate": 3.3333333333333335e-05,
|
|
"loss": 0.0138,
|
|
"num_tokens": 381047.0,
|
|
"reward": 1.1595714092254639,
|
|
"reward_std": 0.08464870601892471,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.30360713601112366,
|
|
"rewards/belief_accuracy/std": 0.02821623906493187,
|
|
"rewards/env_reward/mean": 0.13249999284744263,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 161
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.5,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.135,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.1529526710510254,
|
|
"kl": 0.4582265689969063,
|
|
"learning_rate": 3.3194444444444444e-05,
|
|
"loss": 0.0183,
|
|
"num_tokens": 383509.0,
|
|
"reward": 1.349330186843872,
|
|
"reward_std": 0.10180103778839111,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3788100481033325,
|
|
"rewards/belief_accuracy/std": 0.03024062141776085,
|
|
"rewards/env_reward/mean": 0.10860000550746918,
|
|
"rewards/env_reward/std": 0.009930425323545933,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 162
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.13583333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 1.7175657749176025,
|
|
"kl": 0.37873465567827225,
|
|
"learning_rate": 3.3055555555555553e-05,
|
|
"loss": 0.0151,
|
|
"num_tokens": 385625.0,
|
|
"reward": 1.6891059875488281,
|
|
"reward_std": 0.6854898929595947,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.328631192445755,
|
|
"rewards/belief_accuracy/std": 0.024180419743061066,
|
|
"rewards/env_reward/mean": 0.4354749917984009,
|
|
"rewards/env_reward/std": 0.4491499960422516,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 163
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.13666666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.273981094360352,
|
|
"kl": 0.3159671910107136,
|
|
"learning_rate": 3.291666666666667e-05,
|
|
"loss": 0.0126,
|
|
"num_tokens": 388065.0,
|
|
"reward": 2.1328177452087402,
|
|
"reward_std": 0.10624248534440994,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.44292253255844116,
|
|
"rewards/belief_accuracy/std": 0.035414163023233414,
|
|
"rewards/env_reward/mean": 0.5026999711990356,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 164
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1375,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.6132190227508545,
|
|
"kl": 0.4029811918735504,
|
|
"learning_rate": 3.277777777777778e-05,
|
|
"loss": 0.0161,
|
|
"num_tokens": 390505.0,
|
|
"reward": 0.6693507432937622,
|
|
"reward_std": 0.06415002793073654,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.29170024394989014,
|
|
"rewards/belief_accuracy/std": 0.02138334885239601,
|
|
"rewards/env_reward/mean": -0.1704999953508377,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 17.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 17.0,
|
|
"completions/max_terminated_length": 17.0,
|
|
"completions/mean_length": 12.75,
|
|
"completions/mean_terminated_length": 12.75,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.13833333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.097934246063232,
|
|
"kl": 1.0165367349982262,
|
|
"learning_rate": 3.263888888888889e-05,
|
|
"loss": 0.0407,
|
|
"num_tokens": 392956.0,
|
|
"reward": 1.1792399883270264,
|
|
"reward_std": 0.04536087065935135,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.35646331310272217,
|
|
"rewards/belief_accuracy/std": 0.015120310708880424,
|
|
"rewards/env_reward/mean": 0.039900001138448715,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 166
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 11.0,
|
|
"completions/mean_terminated_length": 11.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.13916666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.14641809463501,
|
|
"kl": 0.4241368919610977,
|
|
"learning_rate": 3.2500000000000004e-05,
|
|
"loss": 0.017,
|
|
"num_tokens": 395400.0,
|
|
"reward": 1.125349521636963,
|
|
"reward_std": 0.05555558204650879,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.30679982900619507,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.10329999774694443,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 167
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 21.0,
|
|
"completions/mean_length": 18.25,
|
|
"completions/mean_terminated_length": 13.666666984558105,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.14,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.288893222808838,
|
|
"kl": 0.6163829118013382,
|
|
"learning_rate": 3.236111111111111e-05,
|
|
"loss": 0.0247,
|
|
"num_tokens": 397873.0,
|
|
"reward": 1.3380489349365234,
|
|
"reward_std": 0.051960092037916183,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2795746326446533,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.29954999685287476,
|
|
"rewards/env_reward/std": 0.03827832639217377,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 168
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 22.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 22.0,
|
|
"completions/max_terminated_length": 22.0,
|
|
"completions/mean_length": 14.0,
|
|
"completions/mean_terminated_length": 14.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.14083333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.591170787811279,
|
|
"kl": 0.5263254791498184,
|
|
"learning_rate": 3.222222222222223e-05,
|
|
"loss": 0.0211,
|
|
"num_tokens": 400329.0,
|
|
"reward": 1.7974923849105835,
|
|
"reward_std": 0.030818266794085503,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3927474319934845,
|
|
"rewards/belief_accuracy/std": 0.010272752493619919,
|
|
"rewards/env_reward/mean": 0.37950000166893005,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 169
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 11.0,
|
|
"completions/mean_terminated_length": 11.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.14166666666666666,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 2.159360885620117,
|
|
"kl": 0.7782266139984131,
|
|
"learning_rate": 3.208333333333334e-05,
|
|
"loss": 0.0311,
|
|
"num_tokens": 402773.0,
|
|
"reward": 1.3888577222824097,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.34553590416908264,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.20149999856948853,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 26.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 26.0,
|
|
"completions/max_terminated_length": 26.0,
|
|
"completions/mean_length": 14.0,
|
|
"completions/mean_terminated_length": 14.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1425,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 2.983961343765259,
|
|
"kl": 1.172668918967247,
|
|
"learning_rate": 3.194444444444444e-05,
|
|
"loss": 0.0469,
|
|
"num_tokens": 405229.0,
|
|
"reward": 1.5690133571624756,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.4289877712726593,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.15469999611377716,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 171
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.14333333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.0448641777038574,
|
|
"kl": 0.31760095804929733,
|
|
"learning_rate": 3.180555555555556e-05,
|
|
"loss": 0.0127,
|
|
"num_tokens": 407669.0,
|
|
"reward": 1.2858712673187256,
|
|
"reward_std": 0.013649980537593365,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3892820477485657,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.04535000026226044,
|
|
"rewards/env_reward/std": 0.009099999442696571,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 172
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 17.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 17.0,
|
|
"completions/max_terminated_length": 17.0,
|
|
"completions/mean_length": 11.75,
|
|
"completions/mean_terminated_length": 11.75,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.14416666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 9.481169700622559,
|
|
"kl": 1.9302819445729256,
|
|
"learning_rate": 3.1666666666666666e-05,
|
|
"loss": 0.0772,
|
|
"num_tokens": 410116.0,
|
|
"reward": 1.5479769706726074,
|
|
"reward_std": 0.05555546283721924,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.32457566261291504,
|
|
"rewards/belief_accuracy/std": 0.018518507480621338,
|
|
"rewards/env_reward/mean": 0.34950000047683716,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 173
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 12.0,
|
|
"completions/mean_terminated_length": 12.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.145,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.64962100982666,
|
|
"kl": 0.6141732186079025,
|
|
"learning_rate": 3.1527777777777775e-05,
|
|
"loss": 0.0246,
|
|
"num_tokens": 412564.0,
|
|
"reward": 2.0601048469543457,
|
|
"reward_std": 0.4084498882293701,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3570099174976349,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.6260499954223633,
|
|
"rewards/env_reward/std": 0.27230000495910645,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 174
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 12.0,
|
|
"completions/mean_terminated_length": 12.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.14583333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 19.281583786010742,
|
|
"kl": 0.6186791881918907,
|
|
"learning_rate": 3.138888888888889e-05,
|
|
"loss": 0.0247,
|
|
"num_tokens": 415012.0,
|
|
"reward": 1.3272333145141602,
|
|
"reward_std": 0.06415006518363953,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2631944417953491,
|
|
"rewards/belief_accuracy/std": 0.02138334885239601,
|
|
"rewards/env_reward/mean": 0.32510000467300415,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.14666666666666667,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0038161983247846365,
|
|
"kl": 0.32898589968681335,
|
|
"learning_rate": 3.125e-05,
|
|
"loss": 0.0132,
|
|
"num_tokens": 417452.0,
|
|
"reward": 1.3439536094665527,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.26061785221099854,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.34139999747276306,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 176
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1475,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.02249019406735897,
|
|
"kl": 0.5504751205444336,
|
|
"learning_rate": 3.111111111111111e-05,
|
|
"loss": 0.022,
|
|
"num_tokens": 419892.0,
|
|
"reward": 1.3552190065383911,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.42342302203178406,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.02329999953508377,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 177
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.14833333333333334,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.004471380263566971,
|
|
"kl": 0.3276918828487396,
|
|
"learning_rate": 3.0972222222222226e-05,
|
|
"loss": 0.0131,
|
|
"num_tokens": 422332.0,
|
|
"reward": 1.6074572801589966,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3073524236679077,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.4235999882221222,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 178
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 11.0,
|
|
"completions/mean_terminated_length": 11.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.14916666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 12.483266830444336,
|
|
"kl": 0.5065018609166145,
|
|
"learning_rate": 3.0833333333333335e-05,
|
|
"loss": 0.0203,
|
|
"num_tokens": 424776.0,
|
|
"reward": 1.071094274520874,
|
|
"reward_std": 0.012580275535583496,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2935647964477539,
|
|
"rewards/belief_accuracy/std": 0.004193440079689026,
|
|
"rewards/env_reward/mean": 0.09359999746084213,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 179
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.5,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 21.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.15,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.9969475865364075,
|
|
"kl": 0.444540049880743,
|
|
"learning_rate": 3.069444444444445e-05,
|
|
"loss": 0.0178,
|
|
"num_tokens": 427260.0,
|
|
"reward": 1.8959006071090698,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.38445019721984863,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.4616999924182892,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 13.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 13.0,
|
|
"completions/max_terminated_length": 13.0,
|
|
"completions/mean_length": 10.75,
|
|
"completions/mean_terminated_length": 10.75,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.15083333333333335,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 10.70042610168457,
|
|
"kl": 1.6388273686170578,
|
|
"learning_rate": 3.055555555555556e-05,
|
|
"loss": 0.0656,
|
|
"num_tokens": 429703.0,
|
|
"reward": 1.3662413358688354,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.4158470928668976,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.04580000042915344,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 181
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 14.0,
|
|
"completions/mean_terminated_length": 14.0,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"epoch": 0.15166666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.0670108795166016,
|
|
"kl": 1.061010330915451,
|
|
"learning_rate": 3.0416666666666666e-05,
|
|
"loss": 0.0424,
|
|
"num_tokens": 432159.0,
|
|
"reward": 1.199580192565918,
|
|
"reward_std": 0.059318430721759796,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3253183662891388,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.11574999988079071,
|
|
"rewards/env_reward/std": 0.004099187441170216,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 182
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 11.0,
|
|
"completions/mean_terminated_length": 11.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1525,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.715631484985352,
|
|
"kl": 0.733733519911766,
|
|
"learning_rate": 3.0277777777777776e-05,
|
|
"loss": 0.0293,
|
|
"num_tokens": 434487.0,
|
|
"reward": 1.3816564083099365,
|
|
"reward_std": 0.19201354682445526,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.22851046919822693,
|
|
"rewards/belief_accuracy/std": 0.018518514931201935,
|
|
"rewards/env_reward/mean": 0.4307500123977661,
|
|
"rewards/env_reward/std": 0.1354999989271164,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 183
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 11.0,
|
|
"completions/mean_terminated_length": 11.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.15333333333333332,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 3.4244911670684814,
|
|
"kl": 0.615076094865799,
|
|
"learning_rate": 3.0138888888888888e-05,
|
|
"loss": 0.0246,
|
|
"num_tokens": 436931.0,
|
|
"reward": 1.1164329051971436,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.33457762002944946,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.04179999977350235,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 184
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 18.5,
|
|
"completions/mean_terminated_length": 14.0,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"epoch": 0.15416666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 7.267205238342285,
|
|
"kl": 0.7707809656858444,
|
|
"learning_rate": 3e-05,
|
|
"loss": 0.0308,
|
|
"num_tokens": 439405.0,
|
|
"reward": 0.9776316285133362,
|
|
"reward_std": 0.05555548146367073,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3959105610847473,
|
|
"rewards/belief_accuracy/std": 0.018518507480621338,
|
|
"rewards/env_reward/mean": -0.17339999973773956,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.155,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.635013103485107,
|
|
"kl": 0.3450919836759567,
|
|
"learning_rate": 2.9861111111111113e-05,
|
|
"loss": 0.0138,
|
|
"num_tokens": 441845.0,
|
|
"reward": 1.654306173324585,
|
|
"reward_std": 0.06415006518363953,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.40296870470046997,
|
|
"rewards/belief_accuracy/std": 0.02138334885239601,
|
|
"rewards/env_reward/mean": 0.2635999917984009,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 186
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.5,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.15583333333333332,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 409.9649963378906,
|
|
"kl": 0.933860570192337,
|
|
"learning_rate": 2.9722222222222223e-05,
|
|
"loss": 0.0374,
|
|
"num_tokens": 444307.0,
|
|
"reward": 1.4882175922393799,
|
|
"reward_std": 0.005075931549072266,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.310455858707428,
|
|
"rewards/belief_accuracy/std": 0.0016919821500778198,
|
|
"rewards/env_reward/mean": 0.3379000127315521,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 187
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 14.0,
|
|
"completions/mean_terminated_length": 14.0,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"epoch": 0.15666666666666668,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.20684058964252472,
|
|
"kl": 1.212807685136795,
|
|
"learning_rate": 2.9583333333333335e-05,
|
|
"loss": 0.0485,
|
|
"num_tokens": 446763.0,
|
|
"reward": 1.4152576923370361,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.34553590416908264,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.2190999984741211,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 188
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1575,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.166232109069824,
|
|
"kl": 0.45745784789323807,
|
|
"learning_rate": 2.9444444444444448e-05,
|
|
"loss": 0.0183,
|
|
"num_tokens": 448695.0,
|
|
"reward": 1.4528826475143433,
|
|
"reward_std": 0.2088000327348709,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.40842756628990173,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.11839999258518219,
|
|
"rewards/env_reward/std": 0.13920000195503235,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 189
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 14.0,
|
|
"completions/mean_terminated_length": 14.0,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"epoch": 0.15833333333333333,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.026986636221408844,
|
|
"kl": 1.0310405790805817,
|
|
"learning_rate": 2.9305555555555557e-05,
|
|
"loss": 0.0412,
|
|
"num_tokens": 451151.0,
|
|
"reward": 0.7838923931121826,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.22358080744743347,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.042100001126527786,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.15916666666666668,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.072092056274414,
|
|
"kl": 0.402814045548439,
|
|
"learning_rate": 2.916666666666667e-05,
|
|
"loss": 0.0161,
|
|
"num_tokens": 453591.0,
|
|
"reward": 1.2498445510864258,
|
|
"reward_std": 0.1845216602087021,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.4303731918334961,
|
|
"rewards/belief_accuracy/std": 0.018518507480621338,
|
|
"rewards/env_reward/mean": -0.060850001871585846,
|
|
"rewards/env_reward/std": 0.13030001521110535,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 191
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 16.5,
|
|
"completions/mean_terminated_length": 11.333333969116211,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.16,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 4.364622116088867,
|
|
"kl": 0.520959883928299,
|
|
"learning_rate": 2.9027777777777782e-05,
|
|
"loss": 0.0208,
|
|
"num_tokens": 456057.0,
|
|
"reward": 1.8143787384033203,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.28782621026039124,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.600600004196167,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 192
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 18.5,
|
|
"completions/mean_terminated_length": 14.0,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"epoch": 0.16083333333333333,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 6.7475361824035645,
|
|
"kl": 0.5597751773893833,
|
|
"learning_rate": 2.8888888888888888e-05,
|
|
"loss": 0.0224,
|
|
"num_tokens": 458531.0,
|
|
"reward": -0.030727386474609375,
|
|
"reward_std": 3.4795150756835938,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.23410753905773163,
|
|
"rewards/belief_accuracy/std": 0.2894050180912018,
|
|
"rewards/env_reward/mean": -0.4886999726295471,
|
|
"rewards/env_reward/std": 1.6742000579833984,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 193
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 19.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 19.0,
|
|
"completions/max_terminated_length": 19.0,
|
|
"completions/mean_length": 15.25,
|
|
"completions/mean_terminated_length": 15.25,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"epoch": 0.16166666666666665,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.9581692218780518,
|
|
"kl": 0.9280332177877426,
|
|
"learning_rate": 2.8749999999999997e-05,
|
|
"loss": 0.0371,
|
|
"num_tokens": 460992.0,
|
|
"reward": 2.1100287437438965,
|
|
"reward_std": 0.2535001337528229,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.28782621026039124,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.7976999878883362,
|
|
"rewards/env_reward/std": 0.1690000295639038,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 194
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 14.0,
|
|
"completions/mean_terminated_length": 14.0,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"epoch": 0.1625,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.230289936065674,
|
|
"kl": 0.9568361341953278,
|
|
"learning_rate": 2.861111111111111e-05,
|
|
"loss": 0.0383,
|
|
"num_tokens": 463448.0,
|
|
"reward": 1.7242618799209595,
|
|
"reward_std": 0.059025008231401443,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.46602481603622437,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.18412499129772186,
|
|
"rewards/env_reward/std": 0.0393499955534935,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 16.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 16.0,
|
|
"completions/max_terminated_length": 16.0,
|
|
"completions/mean_length": 13.0,
|
|
"completions/mean_terminated_length": 13.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.16333333333333333,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.7262094616889954,
|
|
"kl": 0.7856011986732483,
|
|
"learning_rate": 2.8472222222222223e-05,
|
|
"loss": 0.0314,
|
|
"num_tokens": 465900.0,
|
|
"reward": 0.789142370223999,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.22358080744743347,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.04560000076889992,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 196
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.16416666666666666,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.02980131469666958,
|
|
"kl": 0.49150124192237854,
|
|
"learning_rate": 2.8333333333333335e-05,
|
|
"loss": 0.0197,
|
|
"num_tokens": 468340.0,
|
|
"reward": 1.2274935245513916,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.269497811794281,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.2460000067949295,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 197
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 11.0,
|
|
"completions/mean_terminated_length": 11.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.165,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.6604632139205933,
|
|
"kl": 0.6369934007525444,
|
|
"learning_rate": 2.8194444444444445e-05,
|
|
"loss": 0.0255,
|
|
"num_tokens": 470784.0,
|
|
"reward": 1.088317632675171,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.4617558717727661,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": -0.2312999963760376,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 198
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 30.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 30.0,
|
|
"completions/max_terminated_length": 30.0,
|
|
"completions/mean_length": 18.0,
|
|
"completions/mean_terminated_length": 18.0,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"epoch": 0.16583333333333333,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 1.6423118114471436,
|
|
"kl": 1.3361373841762543,
|
|
"learning_rate": 2.8055555555555557e-05,
|
|
"loss": 0.0534,
|
|
"num_tokens": 473256.0,
|
|
"reward": 1.2390079498291016,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.386385977268219,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.019899999722838402,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 199
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 12.0,
|
|
"completions/mean_terminated_length": 12.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.16666666666666666,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.3381861746311188,
|
|
"kl": 0.7480977475643158,
|
|
"learning_rate": 2.791666666666667e-05,
|
|
"loss": 0.0299,
|
|
"num_tokens": 475704.0,
|
|
"reward": 1.2790579795837402,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.386385977268219,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.04659999907016754,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 18.5,
|
|
"completions/mean_terminated_length": 14.0,
|
|
"completions/min_length": 14.0,
|
|
"completions/min_terminated_length": 14.0,
|
|
"epoch": 0.1675,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 3.254331588745117,
|
|
"kl": 1.571349710226059,
|
|
"learning_rate": 2.777777777777778e-05,
|
|
"loss": 0.0629,
|
|
"num_tokens": 478178.0,
|
|
"reward": 1.3384069204330444,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.37561896443367004,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.10769999772310257,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 201
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 13.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 13.0,
|
|
"completions/max_terminated_length": 13.0,
|
|
"completions/mean_length": 10.75,
|
|
"completions/mean_terminated_length": 10.75,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.16833333333333333,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.34370848536491394,
|
|
"kl": 0.48092682659626007,
|
|
"learning_rate": 2.7638888888888892e-05,
|
|
"loss": 0.0192,
|
|
"num_tokens": 480621.0,
|
|
"reward": 1.2113056182861328,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3113018572330475,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.15160000324249268,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 202
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.5,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.16916666666666666,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 21.64349937438965,
|
|
"kl": 0.2917898967862129,
|
|
"learning_rate": 2.7500000000000004e-05,
|
|
"loss": 0.0117,
|
|
"num_tokens": 483083.0,
|
|
"reward": 1.6178982257843018,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.29648277163505554,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.4523000121116638,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 203
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.17,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.006457047536969185,
|
|
"kl": 0.4465770423412323,
|
|
"learning_rate": 2.7361111111111114e-05,
|
|
"loss": 0.0179,
|
|
"num_tokens": 485523.0,
|
|
"reward": 1.2813693284988403,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.39455646276474,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.03180000185966492,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 204
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 17.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 17.0,
|
|
"completions/max_terminated_length": 17.0,
|
|
"completions/mean_length": 11.5,
|
|
"completions/mean_terminated_length": 11.5,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.17083333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 2.0349791049957275,
|
|
"kl": 0.9023318737745285,
|
|
"learning_rate": 2.7222222222222223e-05,
|
|
"loss": 0.0361,
|
|
"num_tokens": 487829.0,
|
|
"reward": 1.9587321281433105,
|
|
"reward_std": 0.12462284415960312,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3476565480232239,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.5771750211715698,
|
|
"rewards/env_reward/std": 0.09858879446983337,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 205
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 20.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 20.0,
|
|
"completions/max_terminated_length": 20.0,
|
|
"completions/mean_length": 12.5,
|
|
"completions/mean_terminated_length": 12.5,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.17166666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 17.28134536743164,
|
|
"kl": 0.6101161614060402,
|
|
"learning_rate": 2.7083333333333332e-05,
|
|
"loss": 0.0244,
|
|
"num_tokens": 490279.0,
|
|
"reward": 1.4633797407150269,
|
|
"reward_std": 0.007893800735473633,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.44797658920288086,
|
|
"rewards/belief_accuracy/std": 0.002631261944770813,
|
|
"rewards/env_reward/mean": 0.046300001442432404,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 206
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1725,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.001407866133376956,
|
|
"kl": 0.5468098521232605,
|
|
"learning_rate": 2.6944444444444445e-05,
|
|
"loss": 0.0219,
|
|
"num_tokens": 492719.0,
|
|
"reward": 1.327993392944336,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.269497811794281,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.31299999356269836,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 207
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 19.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 19.0,
|
|
"completions/max_terminated_length": 19.0,
|
|
"completions/mean_length": 12.25,
|
|
"completions/mean_terminated_length": 12.25,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.17333333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.544673919677734,
|
|
"kl": 0.5928681045770645,
|
|
"learning_rate": 2.6805555555555557e-05,
|
|
"loss": 0.0237,
|
|
"num_tokens": 495028.0,
|
|
"reward": 1.942160725593567,
|
|
"reward_std": 0.36202511191368103,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.29648277163505554,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.6684750318527222,
|
|
"rewards/env_reward/std": 0.24135003983974457,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 208
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 26.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 26.0,
|
|
"completions/max_terminated_length": 26.0,
|
|
"completions/mean_length": 15.0,
|
|
"completions/mean_terminated_length": 15.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.17416666666666666,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.25935015082359314,
|
|
"kl": 0.5315433144569397,
|
|
"learning_rate": 2.6666666666666667e-05,
|
|
"loss": 0.0213,
|
|
"num_tokens": 497488.0,
|
|
"reward": 2.0421676635742188,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.4617558717727661,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.40459999442100525,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 209
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.175,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.167694091796875,
|
|
"kl": 0.49413691461086273,
|
|
"learning_rate": 2.652777777777778e-05,
|
|
"loss": 0.0198,
|
|
"num_tokens": 499928.0,
|
|
"reward": 1.8865565061569214,
|
|
"reward_std": 0.05555558204650879,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2970854640007019,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.6302000284194946,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.17583333333333334,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.009997401386499405,
|
|
"kl": 0.4289768636226654,
|
|
"learning_rate": 2.6388888888888892e-05,
|
|
"loss": 0.0172,
|
|
"num_tokens": 502368.0,
|
|
"reward": 2.1709790229797363,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3181929886341095,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.7775999903678894,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 211
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.17666666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.784295558929443,
|
|
"kl": 0.4447771683335304,
|
|
"learning_rate": 2.625e-05,
|
|
"loss": 0.0178,
|
|
"num_tokens": 504808.0,
|
|
"reward": 1.6073195934295654,
|
|
"reward_std": 0.05555550381541252,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3476565480232239,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.34290000796318054,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 212
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1775,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.017436975613236427,
|
|
"kl": 0.4346160590648651,
|
|
"learning_rate": 2.6111111111111114e-05,
|
|
"loss": 0.0174,
|
|
"num_tokens": 507248.0,
|
|
"reward": 1.607659101486206,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2377697229385376,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.5629000067710876,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 213
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.5,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.17833333333333334,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 12.183658599853516,
|
|
"kl": 0.6210581138730049,
|
|
"learning_rate": 2.5972222222222226e-05,
|
|
"loss": 0.0248,
|
|
"num_tokens": 509202.0,
|
|
"reward": -0.27445781230926514,
|
|
"reward_std": 3.327219009399414,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.18266406655311584,
|
|
"rewards/belief_accuracy/std": 0.25510939955711365,
|
|
"rewards/env_reward/mean": -0.5482999682426453,
|
|
"rewards/env_reward/std": 1.6436470746994019,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 214
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.17916666666666667,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0025398649740964174,
|
|
"kl": 0.5393896698951721,
|
|
"learning_rate": 2.5833333333333336e-05,
|
|
"loss": 0.0216,
|
|
"num_tokens": 511642.0,
|
|
"reward": 0.9280062317848206,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3102187514305115,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": -0.035100001841783524,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.18,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.7643086910247803,
|
|
"kl": 0.5103324726223946,
|
|
"learning_rate": 2.5694444444444445e-05,
|
|
"loss": 0.0204,
|
|
"num_tokens": 514082.0,
|
|
"reward": 1.2094720602035522,
|
|
"reward_std": 0.05555558204650879,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.34864068031311035,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.07569999992847443,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 216
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 15.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 15.0,
|
|
"completions/max_terminated_length": 15.0,
|
|
"completions/mean_length": 11.75,
|
|
"completions/mean_terminated_length": 11.75,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.18083333333333335,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.647409439086914,
|
|
"kl": 0.8914579898118973,
|
|
"learning_rate": 2.5555555555555554e-05,
|
|
"loss": 0.0357,
|
|
"num_tokens": 515817.0,
|
|
"reward": 1.9434584379196167,
|
|
"reward_std": 0.26885563135147095,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.41240280866622925,
|
|
"rewards/belief_accuracy/std": 0.046603914350271225,
|
|
"rewards/env_reward/mean": 0.4375,
|
|
"rewards/env_reward/std": 0.22679999470710754,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 217
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.18166666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.6908979415893555,
|
|
"kl": 0.5759952142834663,
|
|
"learning_rate": 2.5416666666666667e-05,
|
|
"loss": 0.023,
|
|
"num_tokens": 518257.0,
|
|
"reward": 1.0635591745376587,
|
|
"reward_std": 0.055555541068315506,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3233530819416046,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.028999999165534973,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 218
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 11.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 11.0,
|
|
"completions/max_terminated_length": 11.0,
|
|
"completions/mean_length": 10.25,
|
|
"completions/mean_terminated_length": 10.25,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1825,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 5.2558979988098145,
|
|
"kl": 0.39761848002672195,
|
|
"learning_rate": 2.527777777777778e-05,
|
|
"loss": 0.0159,
|
|
"num_tokens": 520698.0,
|
|
"reward": 1.4361964464187622,
|
|
"reward_std": 0.05555558204650879,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.44556546211242676,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.032999999821186066,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 219
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 15.5,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.18333333333333332,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 13.562455177307129,
|
|
"kl": 0.3410550318658352,
|
|
"learning_rate": 2.513888888888889e-05,
|
|
"loss": 0.0136,
|
|
"num_tokens": 523160.0,
|
|
"reward": 1.1828861236572266,
|
|
"reward_std": 0.05555546283721924,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.366778701543808,
|
|
"rewards/belief_accuracy/std": 0.018518507480621338,
|
|
"rewards/env_reward/mean": 0.021700000390410423,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 16.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 16.0,
|
|
"completions/max_terminated_length": 16.0,
|
|
"completions/mean_length": 11.5,
|
|
"completions/mean_terminated_length": 11.5,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.18416666666666667,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 31.85221290588379,
|
|
"kl": 1.1215265393257141,
|
|
"learning_rate": 2.5e-05,
|
|
"loss": 0.0449,
|
|
"num_tokens": 525606.0,
|
|
"reward": 1.623335361480713,
|
|
"reward_std": 0.07827307283878326,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.43429505825042725,
|
|
"rewards/belief_accuracy/std": 0.026091037318110466,
|
|
"rewards/env_reward/mean": 0.18029999732971191,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 221
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.185,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.04103556647896767,
|
|
"kl": 0.6520499587059021,
|
|
"learning_rate": 2.4861111111111114e-05,
|
|
"loss": 0.0261,
|
|
"num_tokens": 528046.0,
|
|
"reward": 1.719389796257019,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3248632550239563,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.46320000290870667,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 222
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 32.0,
|
|
"completions/clipped_ratio": 0.25,
|
|
"completions/max_length": 32.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 16.5,
|
|
"completions/mean_terminated_length": 11.333333969116211,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.18583333333333332,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 25.01701545715332,
|
|
"kl": 0.5154426582157612,
|
|
"learning_rate": 2.4722222222222223e-05,
|
|
"loss": 0.0206,
|
|
"num_tokens": 530512.0,
|
|
"reward": 1.1061065196990967,
|
|
"reward_std": 0.09622509032487869,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3286855220794678,
|
|
"rewards/belief_accuracy/std": 0.03207503259181976,
|
|
"rewards/env_reward/mean": 0.04670000076293945,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 223
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 18.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 18.0,
|
|
"completions/max_terminated_length": 18.0,
|
|
"completions/mean_length": 14.5,
|
|
"completions/mean_terminated_length": 14.5,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.18666666666666668,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 21.70404052734375,
|
|
"kl": 0.6355895921587944,
|
|
"learning_rate": 2.4583333333333332e-05,
|
|
"loss": 0.0254,
|
|
"num_tokens": 532970.0,
|
|
"reward": 1.7521222829818726,
|
|
"reward_std": 0.05555558204650879,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.44232407212257385,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.2500999867916107,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 224
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1875,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.6001179218292236,
|
|
"kl": 0.509816125035286,
|
|
"learning_rate": 2.4444444444444445e-05,
|
|
"loss": 0.0204,
|
|
"num_tokens": 535410.0,
|
|
"reward": 1.56881582736969,
|
|
"reward_std": 0.06415006518363953,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.37087196111679077,
|
|
"rewards/belief_accuracy/std": 0.02138334885239601,
|
|
"rewards/env_reward/mean": 0.27079999446868896,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 225
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 17.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 17.0,
|
|
"completions/max_terminated_length": 17.0,
|
|
"completions/mean_length": 13.0,
|
|
"completions/mean_terminated_length": 13.0,
|
|
"completions/min_length": 9.0,
|
|
"completions/min_terminated_length": 9.0,
|
|
"epoch": 0.18833333333333332,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 4.272892951965332,
|
|
"kl": 0.7432731539011002,
|
|
"learning_rate": 2.4305555555555558e-05,
|
|
"loss": 0.0297,
|
|
"num_tokens": 537862.0,
|
|
"reward": 1.6716521978378296,
|
|
"reward_std": 0.282707542181015,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3669257164001465,
|
|
"rewards/belief_accuracy/std": 0.02138334885239601,
|
|
"rewards/env_reward/mean": 0.34724998474121094,
|
|
"rewards/env_reward/std": 0.20989999175071716,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 226
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.18916666666666668,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.01726493611931801,
|
|
"kl": 0.46494513750076294,
|
|
"learning_rate": 2.4166666666666667e-05,
|
|
"loss": 0.0186,
|
|
"num_tokens": 540302.0,
|
|
"reward": 0.8716050386428833,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.32821834087371826,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": -0.10869999974966049,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 227
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 19.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 19.0,
|
|
"completions/max_terminated_length": 19.0,
|
|
"completions/mean_length": 13.75,
|
|
"completions/mean_terminated_length": 13.75,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.19,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.8162057399749756,
|
|
"kl": 0.9498792588710785,
|
|
"learning_rate": 2.402777777777778e-05,
|
|
"loss": 0.038,
|
|
"num_tokens": 542757.0,
|
|
"reward": 2.149008274078369,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3575194478034973,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.6843000054359436,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 228
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.19083333333333333,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.008917681872844696,
|
|
"kl": 0.48936086893081665,
|
|
"learning_rate": 2.3888888888888892e-05,
|
|
"loss": 0.0196,
|
|
"num_tokens": 545197.0,
|
|
"reward": 1.225246787071228,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3493489623069763,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.08479999750852585,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 229
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 16.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 16.0,
|
|
"completions/max_terminated_length": 16.0,
|
|
"completions/mean_length": 11.75,
|
|
"completions/mean_terminated_length": 11.75,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.19166666666666668,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 10.427980422973633,
|
|
"kl": 0.6280895695090294,
|
|
"learning_rate": 2.375e-05,
|
|
"loss": 0.0251,
|
|
"num_tokens": 547644.0,
|
|
"reward": 1.8005164861679077,
|
|
"reward_std": 0.025754213333129883,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3572555184364319,
|
|
"rewards/belief_accuracy/std": 0.008584737777709961,
|
|
"rewards/env_reward/mean": 0.45249998569488525,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1925,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.009933769702911377,
|
|
"kl": 0.6541983485221863,
|
|
"learning_rate": 2.361111111111111e-05,
|
|
"loss": 0.0262,
|
|
"num_tokens": 550084.0,
|
|
"reward": 1.3975412845611572,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.23698042333126068,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.4244000017642975,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 231
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 11.0,
|
|
"completions/mean_terminated_length": 11.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.19333333333333333,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 7.480578899383545,
|
|
"kl": 1.1603283882141113,
|
|
"learning_rate": 2.3472222222222223e-05,
|
|
"loss": 0.0464,
|
|
"num_tokens": 552528.0,
|
|
"reward": 1.1860084533691406,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3575194478034973,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.04230000078678131,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 232
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 14.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 14.0,
|
|
"completions/max_terminated_length": 14.0,
|
|
"completions/mean_length": 11.0,
|
|
"completions/mean_terminated_length": 11.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.19416666666666665,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.052133798599243,
|
|
"kl": 0.8705815225839615,
|
|
"learning_rate": 2.3333333333333336e-05,
|
|
"loss": 0.0348,
|
|
"num_tokens": 554260.0,
|
|
"reward": 0.02076089382171631,
|
|
"reward_std": 3.515756607055664,
|
|
"rewards/action_legal/mean": -0.25,
|
|
"rewards/action_legal/std": 0.5,
|
|
"rewards/belief_accuracy/mean": 0.21635779738426208,
|
|
"rewards/belief_accuracy/std": 0.28025466203689575,
|
|
"rewards/env_reward/mean": -0.4188750088214874,
|
|
"rewards/env_reward/std": 1.720750093460083,
|
|
"rewards/format_valid/mean": 0.25,
|
|
"rewards/format_valid/std": 1.5,
|
|
"step": 233
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.195,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.004772593267261982,
|
|
"kl": 0.6416133642196655,
|
|
"learning_rate": 2.3194444444444445e-05,
|
|
"loss": 0.0257,
|
|
"num_tokens": 556700.0,
|
|
"reward": 2.1188900470733643,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3552300035953522,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.6687999963760376,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 234
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 23.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 23.0,
|
|
"completions/max_terminated_length": 23.0,
|
|
"completions/mean_length": 13.25,
|
|
"completions/mean_terminated_length": 13.25,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.19583333333333333,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.5717759132385254,
|
|
"kl": 0.6454999148845673,
|
|
"learning_rate": 2.3055555555555558e-05,
|
|
"loss": 0.0258,
|
|
"num_tokens": 559153.0,
|
|
"reward": 2.3321728706359863,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.45852428674697876,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.6043999791145325,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 235
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.19666666666666666,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0032116181682795286,
|
|
"kl": 0.5121519565582275,
|
|
"learning_rate": 2.2916666666666667e-05,
|
|
"loss": 0.0205,
|
|
"num_tokens": 561593.0,
|
|
"reward": 1.2194421291351318,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3623640537261963,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.05490000173449516,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 236
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 20.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 20.0,
|
|
"completions/max_terminated_length": 20.0,
|
|
"completions/mean_length": 12.5,
|
|
"completions/mean_terminated_length": 12.5,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.1975,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 7.710559844970703,
|
|
"kl": 0.8115118741989136,
|
|
"learning_rate": 2.277777777777778e-05,
|
|
"loss": 0.0325,
|
|
"num_tokens": 563535.0,
|
|
"reward": 2.505500316619873,
|
|
"reward_std": 0.45341095328330994,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.24253758788108826,
|
|
"rewards/belief_accuracy/std": 0.03546026349067688,
|
|
"rewards/env_reward/mean": 1.151924967765808,
|
|
"rewards/env_reward/std": 0.281749963760376,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 237
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.19833333333333333,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0054448568262159824,
|
|
"kl": 0.6401026844978333,
|
|
"learning_rate": 2.263888888888889e-05,
|
|
"loss": 0.0256,
|
|
"num_tokens": 565975.0,
|
|
"reward": 1.4314594268798828,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3922864496707916,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.1363999992609024,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 238
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 13.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 13.0,
|
|
"completions/max_terminated_length": 13.0,
|
|
"completions/mean_length": 10.75,
|
|
"completions/mean_terminated_length": 10.75,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.19916666666666666,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.48050761222839355,
|
|
"kl": 0.6103616803884506,
|
|
"learning_rate": 2.25e-05,
|
|
"loss": 0.0244,
|
|
"num_tokens": 568418.0,
|
|
"reward": 1.5885515213012695,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2559005320072174,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.5138999819755554,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 239
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.2,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.010801396332681179,
|
|
"kl": 0.5299305319786072,
|
|
"learning_rate": 2.2361111111111114e-05,
|
|
"loss": 0.0212,
|
|
"num_tokens": 570858.0,
|
|
"reward": 1.7121706008911133,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.44220685958862305,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.22370000183582306,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.20083333333333334,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.005526633467525244,
|
|
"kl": 0.647596538066864,
|
|
"learning_rate": 2.2222222222222223e-05,
|
|
"loss": 0.0259,
|
|
"num_tokens": 573298.0,
|
|
"reward": 1.6188621520996094,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.25235405564308167,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.5411999821662903,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 241
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 16.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 16.0,
|
|
"completions/max_terminated_length": 16.0,
|
|
"completions/mean_length": 11.5,
|
|
"completions/mean_terminated_length": 11.5,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.20166666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 3.6879804134368896,
|
|
"kl": 0.4897921085357666,
|
|
"learning_rate": 2.2083333333333333e-05,
|
|
"loss": 0.0196,
|
|
"num_tokens": 575744.0,
|
|
"reward": 1.4994770288467407,
|
|
"reward_std": 0.05555558204650879,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3609590232372284,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.2443999946117401,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 242
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.2025,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.002229674719274044,
|
|
"kl": 0.5125965476036072,
|
|
"learning_rate": 2.1944444444444445e-05,
|
|
"loss": 0.0205,
|
|
"num_tokens": 578184.0,
|
|
"reward": 1.6194214820861816,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.44499048590660095,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.15629999339580536,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 243
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.20333333333333334,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.019604183733463287,
|
|
"kl": 0.488160103559494,
|
|
"learning_rate": 2.1805555555555558e-05,
|
|
"loss": 0.0195,
|
|
"num_tokens": 580624.0,
|
|
"reward": 1.335284948348999,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2332783341407776,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.3903000056743622,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 244
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 16.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 16.0,
|
|
"completions/max_terminated_length": 16.0,
|
|
"completions/mean_length": 11.5,
|
|
"completions/mean_terminated_length": 11.5,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.20416666666666666,
|
|
"frac_reward_zero_std": 0.0,
|
|
"grad_norm": 12.400581359863281,
|
|
"kl": 0.5316231548786163,
|
|
"learning_rate": 2.1666666666666667e-05,
|
|
"loss": 0.0213,
|
|
"num_tokens": 583070.0,
|
|
"reward": 1.7589713335037231,
|
|
"reward_std": 0.05555558204650879,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.37080714106559753,
|
|
"rewards/belief_accuracy/std": 0.018518522381782532,
|
|
"rewards/env_reward/mean": 0.3977000117301941,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 245
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.205,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.09725174307823181,
|
|
"kl": 0.49825286865234375,
|
|
"learning_rate": 2.152777777777778e-05,
|
|
"loss": 0.0199,
|
|
"num_tokens": 585374.0,
|
|
"reward": 2.471735954284668,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.35569527745246887,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.9031000137329102,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 246
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.20583333333333334,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0035387862008064985,
|
|
"kl": 0.6661872863769531,
|
|
"learning_rate": 2.138888888888889e-05,
|
|
"loss": 0.0266,
|
|
"num_tokens": 587814.0,
|
|
"reward": 1.4029350280761719,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.2332783341407776,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.43540000915527344,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 247
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.20666666666666667,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.028803575783967972,
|
|
"kl": 0.5254324078559875,
|
|
"learning_rate": 2.125e-05,
|
|
"loss": 0.021,
|
|
"num_tokens": 590254.0,
|
|
"reward": 1.569321632385254,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.37139055132865906,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.2700999975204468,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 248
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.2075,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.0019563257228583097,
|
|
"kl": 0.4938497245311737,
|
|
"learning_rate": 2.111111111111111e-05,
|
|
"loss": 0.0198,
|
|
"num_tokens": 592694.0,
|
|
"reward": 1.2491594552993774,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.3335198163986206,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.1324000060558319,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 249
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completion_length": 10.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 10.0,
|
|
"completions/max_terminated_length": 10.0,
|
|
"completions/mean_length": 10.0,
|
|
"completions/mean_terminated_length": 10.0,
|
|
"completions/min_length": 10.0,
|
|
"completions/min_terminated_length": 10.0,
|
|
"epoch": 0.20833333333333334,
|
|
"frac_reward_zero_std": 1.0,
|
|
"grad_norm": 0.028957560658454895,
|
|
"kl": 0.6057292819023132,
|
|
"learning_rate": 2.0972222222222223e-05,
|
|
"loss": 0.0242,
|
|
"num_tokens": 595134.0,
|
|
"reward": 1.4489554166793823,
|
|
"reward_std": 0.0,
|
|
"rewards/action_legal/mean": 0.0,
|
|
"rewards/action_legal/std": 0.0,
|
|
"rewards/belief_accuracy/mean": 0.37641847133636475,
|
|
"rewards/belief_accuracy/std": 0.0,
|
|
"rewards/env_reward/mean": 0.17980000376701355,
|
|
"rewards/env_reward/std": 0.0,
|
|
"rewards/format_valid/mean": 1.0,
|
|
"rewards/format_valid/std": 0.0,
|
|
"step": 250
|
|
}
|
|
],
|
|
"logging_steps": 1,
|
|
"max_steps": 400,
|
|
"num_input_tokens_seen": 595134,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 250,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": false
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 1,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|