1944 lines
72 KiB
JSON
1944 lines
72 KiB
JSON
|
{
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"eval_steps": 50,
|
||
|
|
"global_step": 312,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.10498046875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1526.0,
|
||
|
|
"completions/mean_length": 461.30107421875,
|
||
|
|
"completions/mean_terminated_length": 335.2909790039063,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.016,
|
||
|
|
"grad_norm": 0.00379525707103312,
|
||
|
|
"learning_rate": 3.1249999999999997e-07,
|
||
|
|
"loss": 0.0469,
|
||
|
|
"num_tokens": 19567755.0,
|
||
|
|
"reward": 0.21255219578742982,
|
||
|
|
"reward_std": 0.3932221710681915,
|
||
|
|
"rewards/accuracy_reward": 0.06689453125,
|
||
|
|
"rewards/brier_reward": 0.11327975392341613,
|
||
|
|
"rewards/confidence_one_or_zero": 0.41611328125,
|
||
|
|
"rewards/format_reward": 0.26865234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.5411806344985962,
|
||
|
|
"rewards/smce": -0.474483048915863,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.09873046875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1530.6,
|
||
|
|
"completions/mean_length": 437.7197265625,
|
||
|
|
"completions/mean_terminated_length": 317.39723510742186,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.032,
|
||
|
|
"grad_norm": 0.0042224242351949215,
|
||
|
|
"learning_rate": 6.249999999999999e-07,
|
||
|
|
"loss": 0.0463,
|
||
|
|
"num_tokens": 39150357.0,
|
||
|
|
"reward": 0.22614014744758607,
|
||
|
|
"reward_std": 0.39391623735427855,
|
||
|
|
"rewards/accuracy_reward": 0.062109375,
|
||
|
|
"rewards/brier_reward": 0.12040608525276184,
|
||
|
|
"rewards/confidence_one_or_zero": 0.4322265625,
|
||
|
|
"rewards/format_reward": 0.29501953125,
|
||
|
|
"rewards/mean_confidence_reward": 0.5668324828147888,
|
||
|
|
"rewards/smce": -0.505133843421936,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.05341796875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1478.6,
|
||
|
|
"completions/mean_length": 311.09013671875,
|
||
|
|
"completions/mean_terminated_length": 242.4318878173828,
|
||
|
|
"completions/min_length": 1.4,
|
||
|
|
"completions/min_terminated_length": 1.4,
|
||
|
|
"epoch": 0.048,
|
||
|
|
"grad_norm": 0.00403438089415431,
|
||
|
|
"learning_rate": 9.374999999999999e-07,
|
||
|
|
"loss": 0.0477,
|
||
|
|
"num_tokens": 57384656.0,
|
||
|
|
"reward": 0.38819911479949953,
|
||
|
|
"reward_std": 0.43736083507537843,
|
||
|
|
"rewards/accuracy_reward": 0.10380859375,
|
||
|
|
"rewards/brier_reward": 0.1952836811542511,
|
||
|
|
"rewards/confidence_one_or_zero": 0.53505859375,
|
||
|
|
"rewards/format_reward": 0.5064453125,
|
||
|
|
"rewards/mean_confidence_reward": 0.6857642889022827,
|
||
|
|
"rewards/smce": -0.5828361272811889,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.02236328125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1449.0,
|
||
|
|
"completions/mean_length": 212.86318359375,
|
||
|
|
"completions/mean_terminated_length": 182.72677917480468,
|
||
|
|
"completions/min_length": 14.2,
|
||
|
|
"completions/min_terminated_length": 14.2,
|
||
|
|
"epoch": 0.064,
|
||
|
|
"grad_norm": 0.0026617355179041624,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.035,
|
||
|
|
"num_tokens": 74482775.0,
|
||
|
|
"reward": 0.6544042944908142,
|
||
|
|
"reward_std": 0.40470498204231264,
|
||
|
|
"rewards/accuracy_reward": 0.18193359375,
|
||
|
|
"rewards/brier_reward": 0.3313737392425537,
|
||
|
|
"rewards/confidence_one_or_zero": 0.56455078125,
|
||
|
|
"rewards/format_reward": 0.82607421875,
|
||
|
|
"rewards/mean_confidence_reward": 0.7916394591331481,
|
||
|
|
"rewards/smce": -0.6115134835243226,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0138671875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1197.4,
|
||
|
|
"completions/mean_length": 158.989453125,
|
||
|
|
"completions/mean_terminated_length": 139.62335815429688,
|
||
|
|
"completions/min_length": 17.2,
|
||
|
|
"completions/min_terminated_length": 17.2,
|
||
|
|
"epoch": 0.08,
|
||
|
|
"grad_norm": 0.0025597603525966406,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0179,
|
||
|
|
"num_tokens": 91043979.0,
|
||
|
|
"reward": 0.7784090042114258,
|
||
|
|
"reward_std": 0.330885636806488,
|
||
|
|
"rewards/accuracy_reward": 0.2107421875,
|
||
|
|
"rewards/brier_reward": 0.4119755744934082,
|
||
|
|
"rewards/confidence_one_or_zero": 0.4875,
|
||
|
|
"rewards/format_reward": 0.96240234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.7723959684371948,
|
||
|
|
"rewards/smce": -0.5660923838615417,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00869140625,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1161.8,
|
||
|
|
"completions/mean_length": 142.11708984375,
|
||
|
|
"completions/mean_terminated_length": 129.9005920410156,
|
||
|
|
"completions/min_length": 40.4,
|
||
|
|
"completions/min_terminated_length": 40.4,
|
||
|
|
"epoch": 0.096,
|
||
|
|
"grad_norm": 0.002101193880662322,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.014,
|
||
|
|
"num_tokens": 107543866.0,
|
||
|
|
"reward": 0.8724155187606811,
|
||
|
|
"reward_std": 0.2890653073787689,
|
||
|
|
"rewards/accuracy_reward": 0.23212890625,
|
||
|
|
"rewards/brier_reward": 0.5501087605953217,
|
||
|
|
"rewards/confidence_one_or_zero": 0.3072265625,
|
||
|
|
"rewards/format_reward": 0.98203125,
|
||
|
|
"rewards/mean_confidence_reward": 0.603573477268219,
|
||
|
|
"rewards/smce": -0.38879336416721344,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00419921875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 990.2,
|
||
|
|
"completions/mean_length": 121.371484375,
|
||
|
|
"completions/mean_terminated_length": 115.4102279663086,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"epoch": 0.112,
|
||
|
|
"grad_norm": 0.0015070850495249033,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.008,
|
||
|
|
"num_tokens": 123896182.0,
|
||
|
|
"reward": 0.9859124302864075,
|
||
|
|
"reward_std": 0.20140353739261627,
|
||
|
|
"rewards/accuracy_reward": 0.27421875,
|
||
|
|
"rewards/brier_reward": 0.7134735107421875,
|
||
|
|
"rewards/confidence_one_or_zero": 0.2005859375,
|
||
|
|
"rewards/format_reward": 0.99169921875,
|
||
|
|
"rewards/mean_confidence_reward": 0.35798991322517393,
|
||
|
|
"rewards/smce": -0.15135461539030076,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0017578125,
|
||
|
|
"completions/max_length": 1361.0,
|
||
|
|
"completions/max_terminated_length": 950.4,
|
||
|
|
"completions/mean_length": 113.27216796875,
|
||
|
|
"completions/mean_terminated_length": 110.7666030883789,
|
||
|
|
"completions/min_length": 40.4,
|
||
|
|
"completions/min_terminated_length": 40.4,
|
||
|
|
"epoch": 0.128,
|
||
|
|
"grad_norm": 0.0008679942111484706,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0046,
|
||
|
|
"num_tokens": 139972761.0,
|
||
|
|
"reward": 1.0169670820236205,
|
||
|
|
"reward_std": 0.1333804875612259,
|
||
|
|
"rewards/accuracy_reward": 0.2720703125,
|
||
|
|
"rewards/brier_reward": 0.7683740258216858,
|
||
|
|
"rewards/confidence_one_or_zero": 0.15654296875,
|
||
|
|
"rewards/format_reward": 0.9966796875,
|
||
|
|
"rewards/mean_confidence_reward": 0.24390380084514618,
|
||
|
|
"rewards/smce": -0.06381280496716499,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1156.4,
|
||
|
|
"completions/max_terminated_length": 758.0,
|
||
|
|
"completions/mean_length": 111.1904296875,
|
||
|
|
"completions/mean_terminated_length": 110.49412536621094,
|
||
|
|
"completions/min_length": 49.8,
|
||
|
|
"completions/min_terminated_length": 49.8,
|
||
|
|
"epoch": 0.144,
|
||
|
|
"grad_norm": 0.0008737383177503943,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"num_tokens": 156061783.0,
|
||
|
|
"reward": 1.0350018739700317,
|
||
|
|
"reward_std": 0.10654668658971786,
|
||
|
|
"rewards/accuracy_reward": 0.32724609375,
|
||
|
|
"rewards/brier_reward": 0.7511680364608765,
|
||
|
|
"rewards/confidence_one_or_zero": 0.10390625,
|
||
|
|
"rewards/format_reward": 0.99775390625,
|
||
|
|
"rewards/mean_confidence_reward": 0.21236729323863984,
|
||
|
|
"rewards/smce": -0.12329771071672439,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00087890625,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 706.4,
|
||
|
|
"completions/mean_length": 114.46494140625,
|
||
|
|
"completions/mean_terminated_length": 113.21427154541016,
|
||
|
|
"completions/min_length": 44.0,
|
||
|
|
"completions/min_terminated_length": 44.0,
|
||
|
|
"epoch": 0.16,
|
||
|
|
"grad_norm": 0.0007549422443844378,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0019,
|
||
|
|
"num_tokens": 172254832.0,
|
||
|
|
"reward": 1.0396341800689697,
|
||
|
|
"reward_std": 0.09648548364639283,
|
||
|
|
"rewards/accuracy_reward": 0.321875,
|
||
|
|
"rewards/brier_reward": 0.7635697841644287,
|
||
|
|
"rewards/confidence_one_or_zero": 0.046484375,
|
||
|
|
"rewards/format_reward": 0.9984375,
|
||
|
|
"rewards/mean_confidence_reward": 0.23609647154808044,
|
||
|
|
"rewards/smce": -0.09228837788105011,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16,
|
||
|
|
"eval_completions/clipped_ratio": 0.002049180327868827,
|
||
|
|
"eval_completions/max_length": 928.0,
|
||
|
|
"eval_completions/max_terminated_length": 374.0,
|
||
|
|
"eval_completions/mean_length": 117.9098014831543,
|
||
|
|
"eval_completions/mean_terminated_length": 115.00321960449219,
|
||
|
|
"eval_completions/min_length": 56.0,
|
||
|
|
"eval_completions/min_terminated_length": 56.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 172254832.0,
|
||
|
|
"eval_reward": 1.0303497314453125,
|
||
|
|
"eval_reward_std": 0.14893124997615814,
|
||
|
|
"eval_rewards/accuracy_reward": 0.26171875,
|
||
|
|
"eval_rewards/brier_reward": 0.8018642067909241,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0234375,
|
||
|
|
"eval_rewards/format_reward": 0.998046875,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.26405274868011475,
|
||
|
|
"eval_rewards/smce": -0.01861918903887272,
|
||
|
|
"eval_runtime": 21.6493,
|
||
|
|
"eval_samples_per_second": 23.095,
|
||
|
|
"eval_steps_per_second": 0.092,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 709.8,
|
||
|
|
"completions/max_terminated_length": 462.6,
|
||
|
|
"completions/mean_length": 114.0119140625,
|
||
|
|
"completions/mean_terminated_length": 113.8730453491211,
|
||
|
|
"completions/min_length": 47.6,
|
||
|
|
"completions/min_terminated_length": 47.6,
|
||
|
|
"epoch": 0.176,
|
||
|
|
"grad_norm": 0.0008791896980255842,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 188659434.0,
|
||
|
|
"reward": 1.0433724403381348,
|
||
|
|
"reward_std": 0.09667095988988876,
|
||
|
|
"rewards/accuracy_reward": 0.30830078125,
|
||
|
|
"rewards/brier_reward": 0.781142795085907,
|
||
|
|
"rewards/confidence_one_or_zero": 0.02275390625,
|
||
|
|
"rewards/format_reward": 0.9994140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.27407650351524354,
|
||
|
|
"rewards/smce": -0.04226556643843651,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 635.2,
|
||
|
|
"completions/max_terminated_length": 415.8,
|
||
|
|
"completions/mean_length": 113.58115234375,
|
||
|
|
"completions/mean_terminated_length": 113.4424545288086,
|
||
|
|
"completions/min_length": 38.2,
|
||
|
|
"completions/min_terminated_length": 38.2,
|
||
|
|
"epoch": 0.192,
|
||
|
|
"grad_norm": 0.0017089458415284753,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0005,
|
||
|
|
"num_tokens": 204637321.0,
|
||
|
|
"reward": 1.0613579511642457,
|
||
|
|
"reward_std": 0.105955308675766,
|
||
|
|
"rewards/accuracy_reward": 0.348828125,
|
||
|
|
"rewards/brier_reward": 0.7770011067390442,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0076171875,
|
||
|
|
"rewards/format_reward": 0.9994140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.3225338637828827,
|
||
|
|
"rewards/smce": -0.050558538734912874,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 826.0,
|
||
|
|
"completions/max_terminated_length": 335.0,
|
||
|
|
"completions/mean_length": 116.15986328125,
|
||
|
|
"completions/mean_terminated_length": 115.88248596191406,
|
||
|
|
"completions/min_length": 56.4,
|
||
|
|
"completions/min_terminated_length": 56.4,
|
||
|
|
"epoch": 0.208,
|
||
|
|
"grad_norm": 0.0009319775272160769,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 220859022.0,
|
||
|
|
"reward": 1.066517448425293,
|
||
|
|
"reward_std": 0.11409244835376739,
|
||
|
|
"rewards/accuracy_reward": 0.36494140625,
|
||
|
|
"rewards/brier_reward": 0.7699201822280883,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0037109375,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.36000165343284607,
|
||
|
|
"rewards/smce": -0.030688709765672683,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1148.6,
|
||
|
|
"completions/max_terminated_length": 409.6,
|
||
|
|
"completions/mean_length": 114.28154296875,
|
||
|
|
"completions/mean_terminated_length": 113.58670349121094,
|
||
|
|
"completions/min_length": 55.0,
|
||
|
|
"completions/min_terminated_length": 55.0,
|
||
|
|
"epoch": 0.224,
|
||
|
|
"grad_norm": 0.0010573499603196979,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.002,
|
||
|
|
"num_tokens": 237182449.0,
|
||
|
|
"reward": 1.0665554523468017,
|
||
|
|
"reward_std": 0.1144440695643425,
|
||
|
|
"rewards/accuracy_reward": 0.36416015625,
|
||
|
|
"rewards/brier_reward": 0.7713935971260071,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00244140625,
|
||
|
|
"rewards/format_reward": 0.99931640625,
|
||
|
|
"rewards/mean_confidence_reward": 0.3705263793468475,
|
||
|
|
"rewards/smce": -0.03520021215081215,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 537.6,
|
||
|
|
"completions/max_terminated_length": 289.6,
|
||
|
|
"completions/mean_length": 112.665234375,
|
||
|
|
"completions/mean_terminated_length": 112.52650146484375,
|
||
|
|
"completions/min_length": 55.4,
|
||
|
|
"completions/min_terminated_length": 55.4,
|
||
|
|
"epoch": 0.24,
|
||
|
|
"grad_norm": 0.0010807751677930355,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0001,
|
||
|
|
"num_tokens": 253587821.0,
|
||
|
|
"reward": 1.0801397800445556,
|
||
|
|
"reward_std": 0.1174217015504837,
|
||
|
|
"rewards/accuracy_reward": 0.40361328125,
|
||
|
|
"rewards/brier_reward": 0.7600827097892762,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0009765625,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.3807018220424652,
|
||
|
|
"rewards/smce": -0.06443720050156117,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 800.0,
|
||
|
|
"completions/max_terminated_length": 363.2,
|
||
|
|
"completions/mean_length": 110.91630859375,
|
||
|
|
"completions/mean_terminated_length": 110.63789825439453,
|
||
|
|
"completions/min_length": 56.8,
|
||
|
|
"completions/min_terminated_length": 56.8,
|
||
|
|
"epoch": 0.256,
|
||
|
|
"grad_norm": 0.0011304247891530395,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 269778420.0,
|
||
|
|
"reward": 1.0721617221832276,
|
||
|
|
"reward_std": 0.11495736688375473,
|
||
|
|
"rewards/accuracy_reward": 0.3783203125,
|
||
|
|
"rewards/brier_reward": 0.7689534902572632,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00068359375,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.3930156409740448,
|
||
|
|
"rewards/smce": -0.05120910629630089,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 342.2,
|
||
|
|
"completions/max_terminated_length": 342.2,
|
||
|
|
"completions/mean_length": 112.69873046875,
|
||
|
|
"completions/mean_terminated_length": 112.69873046875,
|
||
|
|
"completions/min_length": 57.2,
|
||
|
|
"completions/min_terminated_length": 57.2,
|
||
|
|
"epoch": 0.272,
|
||
|
|
"grad_norm": 0.0010486873798072338,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 285898151.0,
|
||
|
|
"reward": 1.0783493280410767,
|
||
|
|
"reward_std": 0.1138525053858757,
|
||
|
|
"rewards/accuracy_reward": 0.39580078125,
|
||
|
|
"rewards/brier_reward": 0.7631247043609619,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.39648603796958926,
|
||
|
|
"rewards/smce": -0.03478789925575256,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 564.8,
|
||
|
|
"completions/max_terminated_length": 317.4,
|
||
|
|
"completions/mean_length": 113.1732421875,
|
||
|
|
"completions/mean_terminated_length": 113.0341781616211,
|
||
|
|
"completions/min_length": 45.2,
|
||
|
|
"completions/min_terminated_length": 45.2,
|
||
|
|
"epoch": 0.288,
|
||
|
|
"grad_norm": 0.0010555179324001074,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0003,
|
||
|
|
"num_tokens": 302015221.0,
|
||
|
|
"reward": 1.0807376623153686,
|
||
|
|
"reward_std": 0.10946670323610305,
|
||
|
|
"rewards/accuracy_reward": 0.39296875,
|
||
|
|
"rewards/brier_reward": 0.7711483836174011,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.3924079716205597,
|
||
|
|
"rewards/smce": -0.04894471056759357,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 674.2,
|
||
|
|
"completions/max_terminated_length": 423.2,
|
||
|
|
"completions/mean_length": 115.8076171875,
|
||
|
|
"completions/mean_terminated_length": 115.66913299560547,
|
||
|
|
"completions/min_length": 55.0,
|
||
|
|
"completions/min_terminated_length": 55.0,
|
||
|
|
"epoch": 0.304,
|
||
|
|
"grad_norm": 0.0011386368423700333,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 318131043.0,
|
||
|
|
"reward": 1.0865533351898193,
|
||
|
|
"reward_std": 0.10848731994628906,
|
||
|
|
"rewards/accuracy_reward": 0.408203125,
|
||
|
|
"rewards/brier_reward": 0.7670220613479615,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.39132454991340637,
|
||
|
|
"rewards/smce": -0.04043112620711327,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 572.8,
|
||
|
|
"completions/max_terminated_length": 326.2,
|
||
|
|
"completions/mean_length": 116.87548828125,
|
||
|
|
"completions/mean_terminated_length": 116.7369140625,
|
||
|
|
"completions/min_length": 58.4,
|
||
|
|
"completions/min_terminated_length": 58.4,
|
||
|
|
"epoch": 0.32,
|
||
|
|
"grad_norm": 0.0009122883784584701,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 334416552.0,
|
||
|
|
"reward": 1.0942379474639892,
|
||
|
|
"reward_std": 0.1046961173415184,
|
||
|
|
"rewards/accuracy_reward": 0.425390625,
|
||
|
|
"rewards/brier_reward": 0.7664251208305359,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.3974118232727051,
|
||
|
|
"rewards/smce": -0.06485869474709034,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 282.0,
|
||
|
|
"eval_completions/max_terminated_length": 282.0,
|
||
|
|
"eval_completions/mean_length": 119.43567276000977,
|
||
|
|
"eval_completions/mean_terminated_length": 119.43567276000977,
|
||
|
|
"eval_completions/min_length": 62.0,
|
||
|
|
"eval_completions/min_terminated_length": 62.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 334416552.0,
|
||
|
|
"eval_reward": 1.0595381259918213,
|
||
|
|
"eval_reward_std": 0.2181749865412712,
|
||
|
|
"eval_rewards/accuracy_reward": 0.34765625,
|
||
|
|
"eval_rewards/brier_reward": 0.7744396030902863,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4042382836341858,
|
||
|
|
"eval_rewards/smce": -0.060410160571336746,
|
||
|
|
"eval_runtime": 11.1966,
|
||
|
|
"eval_samples_per_second": 44.656,
|
||
|
|
"eval_steps_per_second": 0.179,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 598.4,
|
||
|
|
"completions/max_terminated_length": 347.4,
|
||
|
|
"completions/mean_length": 116.05048828125,
|
||
|
|
"completions/mean_terminated_length": 115.91200714111328,
|
||
|
|
"completions/min_length": 56.2,
|
||
|
|
"completions/min_terminated_length": 56.2,
|
||
|
|
"epoch": 0.336,
|
||
|
|
"grad_norm": 0.000933317409362644,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 350327341.0,
|
||
|
|
"reward": 1.0963164806365966,
|
||
|
|
"reward_std": 0.10393176078796387,
|
||
|
|
"rewards/accuracy_reward": 0.42666015625,
|
||
|
|
"rewards/brier_reward": 0.7689025402069092,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.40582754015922545,
|
||
|
|
"rewards/smce": -0.0527496300637722,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 834.4,
|
||
|
|
"completions/max_terminated_length": 328.2,
|
||
|
|
"completions/mean_length": 119.27626953125,
|
||
|
|
"completions/mean_terminated_length": 118.99936370849609,
|
||
|
|
"completions/min_length": 58.2,
|
||
|
|
"completions/min_terminated_length": 58.2,
|
||
|
|
"epoch": 0.352,
|
||
|
|
"grad_norm": 0.0010004864307120442,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 366809146.0,
|
||
|
|
"reward": 1.0733874320983887,
|
||
|
|
"reward_std": 0.1073990985751152,
|
||
|
|
"rewards/accuracy_reward": 0.37646484375,
|
||
|
|
"rewards/brier_reward": 0.7720993638038636,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4040791392326355,
|
||
|
|
"rewards/smce": -0.029942740127444266,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 610.2,
|
||
|
|
"completions/max_terminated_length": 363.6,
|
||
|
|
"completions/mean_length": 123.4861328125,
|
||
|
|
"completions/mean_terminated_length": 123.34794921875,
|
||
|
|
"completions/min_length": 59.4,
|
||
|
|
"completions/min_terminated_length": 59.4,
|
||
|
|
"epoch": 0.368,
|
||
|
|
"grad_norm": 0.0008533200598321855,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0001,
|
||
|
|
"num_tokens": 383139116.0,
|
||
|
|
"reward": 1.0894709587097169,
|
||
|
|
"reward_std": 0.10314081609249115,
|
||
|
|
"rewards/accuracy_reward": 0.4060546875,
|
||
|
|
"rewards/brier_reward": 0.7751178503036499,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4102395951747894,
|
||
|
|
"rewards/smce": -0.038769982382655145,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 594.6,
|
||
|
|
"completions/max_terminated_length": 343.8,
|
||
|
|
"completions/mean_length": 126.59150390625,
|
||
|
|
"completions/mean_terminated_length": 126.45359497070312,
|
||
|
|
"completions/min_length": 59.6,
|
||
|
|
"completions/min_terminated_length": 59.6,
|
||
|
|
"epoch": 0.384,
|
||
|
|
"grad_norm": 0.0008750154520384967,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 399291925.0,
|
||
|
|
"reward": 1.1030329465866089,
|
||
|
|
"reward_std": 0.09855032563209534,
|
||
|
|
"rewards/accuracy_reward": 0.434765625,
|
||
|
|
"rewards/brier_reward": 0.7735632658004761,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.411165726184845,
|
||
|
|
"rewards/smce": -0.0433216568082571,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 576.6,
|
||
|
|
"completions/max_terminated_length": 332.0,
|
||
|
|
"completions/mean_length": 131.9685546875,
|
||
|
|
"completions/mean_terminated_length": 131.83190002441407,
|
||
|
|
"completions/min_length": 61.8,
|
||
|
|
"completions/min_terminated_length": 61.8,
|
||
|
|
"epoch": 0.4,
|
||
|
|
"grad_norm": 0.0009054332040250301,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 415679731.0,
|
||
|
|
"reward": 1.090169644355774,
|
||
|
|
"reward_std": 0.11003399789333343,
|
||
|
|
"rewards/accuracy_reward": 0.401171875,
|
||
|
|
"rewards/brier_reward": 0.7810108900070191,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.42416959404945376,
|
||
|
|
"rewards/smce": -0.03493335172533989,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0005859375,
|
||
|
|
"completions/max_length": 1102.2,
|
||
|
|
"completions/max_terminated_length": 408.4,
|
||
|
|
"completions/mean_length": 136.919921875,
|
||
|
|
"completions/mean_terminated_length": 136.10001220703126,
|
||
|
|
"completions/min_length": 61.0,
|
||
|
|
"completions/min_terminated_length": 61.0,
|
||
|
|
"epoch": 0.416,
|
||
|
|
"grad_norm": 0.0008075532387010753,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0017,
|
||
|
|
"num_tokens": 431962975.0,
|
||
|
|
"reward": 1.0986106395721436,
|
||
|
|
"reward_std": 0.11043254286050797,
|
||
|
|
"rewards/accuracy_reward": 0.41943359375,
|
||
|
|
"rewards/brier_reward": 0.7794132232666016,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9994140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.4245561957359314,
|
||
|
|
"rewards/smce": -0.020808039605617522,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 576.6,
|
||
|
|
"completions/max_terminated_length": 359.8,
|
||
|
|
"completions/mean_length": 137.3033203125,
|
||
|
|
"completions/mean_terminated_length": 137.16665344238282,
|
||
|
|
"completions/min_length": 62.6,
|
||
|
|
"completions/min_terminated_length": 62.6,
|
||
|
|
"epoch": 0.432,
|
||
|
|
"grad_norm": 0.0010306270560249686,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 448383297.0,
|
||
|
|
"reward": 1.1073107242584228,
|
||
|
|
"reward_std": 0.10405029356479645,
|
||
|
|
"rewards/accuracy_reward": 0.443359375,
|
||
|
|
"rewards/brier_reward": 0.7737538814544678,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4363938868045807,
|
||
|
|
"rewards/smce": -0.04594621378928423,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 653.8,
|
||
|
|
"completions/max_terminated_length": 442.0,
|
||
|
|
"completions/mean_length": 137.083984375,
|
||
|
|
"completions/mean_terminated_length": 136.94732055664062,
|
||
|
|
"completions/min_length": 61.2,
|
||
|
|
"completions/min_terminated_length": 61.2,
|
||
|
|
"epoch": 0.448,
|
||
|
|
"grad_norm": 0.0007984083495102823,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 464739837.0,
|
||
|
|
"reward": 1.101181983947754,
|
||
|
|
"reward_std": 0.10962242484092713,
|
||
|
|
"rewards/accuracy_reward": 0.43056640625,
|
||
|
|
"rewards/brier_reward": 0.7736897945404053,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4366616249084473,
|
||
|
|
"rewards/smce": -0.0359076976776123,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 1080.6,
|
||
|
|
"completions/max_terminated_length": 377.0,
|
||
|
|
"completions/mean_length": 137.136328125,
|
||
|
|
"completions/mean_terminated_length": 136.58931884765624,
|
||
|
|
"completions/min_length": 66.0,
|
||
|
|
"completions/min_terminated_length": 66.0,
|
||
|
|
"epoch": 0.464,
|
||
|
|
"grad_norm": 0.0007966695702634752,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"num_tokens": 481314929.0,
|
||
|
|
"reward": 1.079134178161621,
|
||
|
|
"reward_std": 0.10293117165565491,
|
||
|
|
"rewards/accuracy_reward": 0.3755859375,
|
||
|
|
"rewards/brier_reward": 0.7862482190132141,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4163307726383209,
|
||
|
|
"rewards/smce": -0.06156680956482887,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 652.8,
|
||
|
|
"completions/max_terminated_length": 430.8,
|
||
|
|
"completions/mean_length": 134.71728515625,
|
||
|
|
"completions/mean_terminated_length": 134.58053283691407,
|
||
|
|
"completions/min_length": 65.8,
|
||
|
|
"completions/min_terminated_length": 65.8,
|
||
|
|
"epoch": 0.48,
|
||
|
|
"grad_norm": 0.0009112542029470205,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 497742466.0,
|
||
|
|
"reward": 1.1103950500488282,
|
||
|
|
"reward_std": 0.10099990367889404,
|
||
|
|
"rewards/accuracy_reward": 0.4515625,
|
||
|
|
"rewards/brier_reward": 0.772011113166809,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4020026683807373,
|
||
|
|
"rewards/smce": -0.05373247638344765,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.48,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 300.5,
|
||
|
|
"eval_completions/max_terminated_length": 300.5,
|
||
|
|
"eval_completions/mean_length": 135.326904296875,
|
||
|
|
"eval_completions/mean_terminated_length": 135.326904296875,
|
||
|
|
"eval_completions/min_length": 74.0,
|
||
|
|
"eval_completions/min_terminated_length": 74.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 497742466.0,
|
||
|
|
"eval_reward": 1.067869484424591,
|
||
|
|
"eval_reward_std": 0.22196416556835175,
|
||
|
|
"eval_rewards/accuracy_reward": 0.349609375,
|
||
|
|
"eval_rewards/brier_reward": 0.788908839225769,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.40415431559085846,
|
||
|
|
"eval_rewards/smce": -0.05559943988919258,
|
||
|
|
"eval_runtime": 10.8135,
|
||
|
|
"eval_samples_per_second": 46.239,
|
||
|
|
"eval_steps_per_second": 0.185,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 597.0,
|
||
|
|
"completions/max_terminated_length": 354.8,
|
||
|
|
"completions/mean_length": 133.0171875,
|
||
|
|
"completions/mean_terminated_length": 132.8801483154297,
|
||
|
|
"completions/min_length": 53.2,
|
||
|
|
"completions/min_terminated_length": 53.2,
|
||
|
|
"epoch": 0.496,
|
||
|
|
"grad_norm": 0.0009701360249891877,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 514412402.0,
|
||
|
|
"reward": 1.111799955368042,
|
||
|
|
"reward_std": 0.10661143213510513,
|
||
|
|
"rewards/accuracy_reward": 0.4494140625,
|
||
|
|
"rewards/brier_reward": 0.7759701251983643,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.42597988843917844,
|
||
|
|
"rewards/smce": -0.03179615084081888,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 571.6,
|
||
|
|
"completions/max_terminated_length": 331.2,
|
||
|
|
"completions/mean_length": 130.59658203125,
|
||
|
|
"completions/mean_terminated_length": 130.4593994140625,
|
||
|
|
"completions/min_length": 63.0,
|
||
|
|
"completions/min_terminated_length": 63.0,
|
||
|
|
"epoch": 0.512,
|
||
|
|
"grad_norm": 0.0012009345227852464,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 530895375.0,
|
||
|
|
"reward": 1.1192145109176637,
|
||
|
|
"reward_std": 0.11020909100770951,
|
||
|
|
"rewards/accuracy_reward": 0.459765625,
|
||
|
|
"rewards/brier_reward": 0.7806031942367554,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4468603491783142,
|
||
|
|
"rewards/smce": -0.034905324690043926,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 613.2,
|
||
|
|
"completions/max_terminated_length": 359.8,
|
||
|
|
"completions/mean_length": 137.18359375,
|
||
|
|
"completions/mean_terminated_length": 137.04674682617187,
|
||
|
|
"completions/min_length": 65.4,
|
||
|
|
"completions/min_terminated_length": 65.4,
|
||
|
|
"epoch": 0.528,
|
||
|
|
"grad_norm": 0.001150839147157967,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 547329671.0,
|
||
|
|
"reward": 1.1141958951950073,
|
||
|
|
"reward_std": 0.11657649278640747,
|
||
|
|
"rewards/accuracy_reward": 0.44794921875,
|
||
|
|
"rewards/brier_reward": 0.7825037598609924,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.46873701810836793,
|
||
|
|
"rewards/smce": -0.035382162779569626,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 611.6,
|
||
|
|
"completions/max_terminated_length": 386.8,
|
||
|
|
"completions/mean_length": 142.52001953125,
|
||
|
|
"completions/mean_terminated_length": 142.38402099609374,
|
||
|
|
"completions/min_length": 65.6,
|
||
|
|
"completions/min_terminated_length": 65.6,
|
||
|
|
"epoch": 0.544,
|
||
|
|
"grad_norm": 0.0009249963914044201,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 563952660.0,
|
||
|
|
"reward": 1.1343823671340942,
|
||
|
|
"reward_std": 0.12186299562454224,
|
||
|
|
"rewards/accuracy_reward": 0.49697265625,
|
||
|
|
"rewards/brier_reward": 0.773317551612854,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.48627060651779175,
|
||
|
|
"rewards/smce": -0.02662253864109516,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 480.6,
|
||
|
|
"completions/max_terminated_length": 480.6,
|
||
|
|
"completions/mean_length": 146.33427734375,
|
||
|
|
"completions/mean_terminated_length": 146.33427734375,
|
||
|
|
"completions/min_length": 69.8,
|
||
|
|
"completions/min_terminated_length": 69.8,
|
||
|
|
"epoch": 0.56,
|
||
|
|
"grad_norm": 0.000978289870545268,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 580272531.0,
|
||
|
|
"reward": 1.104417610168457,
|
||
|
|
"reward_std": 0.11959323734045028,
|
||
|
|
"rewards/accuracy_reward": 0.4271484375,
|
||
|
|
"rewards/brier_reward": 0.7850151300430298,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.49268749356269836,
|
||
|
|
"rewards/smce": -0.0665838647633791,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1085.0,
|
||
|
|
"completions/max_terminated_length": 499.2,
|
||
|
|
"completions/mean_length": 155.02001953125,
|
||
|
|
"completions/mean_terminated_length": 154.34617614746094,
|
||
|
|
"completions/min_length": 69.4,
|
||
|
|
"completions/min_terminated_length": 69.4,
|
||
|
|
"epoch": 0.576,
|
||
|
|
"grad_norm": 0.0006955881253816187,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"num_tokens": 597046560.0,
|
||
|
|
"reward": 1.10858473777771,
|
||
|
|
"reward_std": 0.10167575627565384,
|
||
|
|
"rewards/accuracy_reward": 0.4353515625,
|
||
|
|
"rewards/brier_reward": 0.7846779823303223,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4585468828678131,
|
||
|
|
"rewards/smce": -0.04745281338691711,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 582.6,
|
||
|
|
"completions/max_terminated_length": 582.6,
|
||
|
|
"completions/mean_length": 158.10908203125,
|
||
|
|
"completions/mean_terminated_length": 158.10908203125,
|
||
|
|
"completions/min_length": 80.2,
|
||
|
|
"completions/min_terminated_length": 80.2,
|
||
|
|
"epoch": 0.592,
|
||
|
|
"grad_norm": 0.0007171398028731346,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 613833309.0,
|
||
|
|
"reward": 1.1120543956756592,
|
||
|
|
"reward_std": 0.091026870906353,
|
||
|
|
"rewards/accuracy_reward": 0.44326171875,
|
||
|
|
"rewards/brier_reward": 0.7823067307472229,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.42848995327949524,
|
||
|
|
"rewards/smce": -0.029207753576338293,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 636.6,
|
||
|
|
"completions/max_terminated_length": 445.0,
|
||
|
|
"completions/mean_length": 161.95458984375,
|
||
|
|
"completions/mean_terminated_length": 161.6872802734375,
|
||
|
|
"completions/min_length": 80.0,
|
||
|
|
"completions/min_terminated_length": 80.0,
|
||
|
|
"epoch": 0.608,
|
||
|
|
"grad_norm": 0.0007596832583658397,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 630491212.0,
|
||
|
|
"reward": 1.117729663848877,
|
||
|
|
"reward_std": 0.09016477465629577,
|
||
|
|
"rewards/accuracy_reward": 0.44638671875,
|
||
|
|
"rewards/brier_reward": 0.7919416666030884,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.40303613543510436,
|
||
|
|
"rewards/smce": -0.053490759804844856,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 806.8,
|
||
|
|
"completions/max_terminated_length": 591.6,
|
||
|
|
"completions/mean_length": 168.21943359375,
|
||
|
|
"completions/mean_terminated_length": 168.085693359375,
|
||
|
|
"completions/min_length": 75.0,
|
||
|
|
"completions/min_terminated_length": 75.0,
|
||
|
|
"epoch": 0.624,
|
||
|
|
"grad_norm": 0.000917185447178781,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 647557683.0,
|
||
|
|
"reward": 1.1099303245544434,
|
||
|
|
"reward_std": 0.09768695533275604,
|
||
|
|
"rewards/accuracy_reward": 0.43203125,
|
||
|
|
"rewards/brier_reward": 0.7900554180145264,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.42108300924301145,
|
||
|
|
"rewards/smce": -0.04258261527866125,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 844.0,
|
||
|
|
"completions/max_terminated_length": 698.8,
|
||
|
|
"completions/mean_length": 168.75458984375,
|
||
|
|
"completions/mean_terminated_length": 168.6212585449219,
|
||
|
|
"completions/min_length": 78.8,
|
||
|
|
"completions/min_terminated_length": 78.8,
|
||
|
|
"epoch": 0.64,
|
||
|
|
"grad_norm": 0.000845664064399898,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 664628418.0,
|
||
|
|
"reward": 1.148032546043396,
|
||
|
|
"reward_std": 0.10199344009160996,
|
||
|
|
"rewards/accuracy_reward": 0.50791015625,
|
||
|
|
"rewards/brier_reward": 0.7912523984909058,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4784619212150574,
|
||
|
|
"rewards/smce": -0.06001344136893749,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.64,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 344.0,
|
||
|
|
"eval_completions/max_terminated_length": 344.0,
|
||
|
|
"eval_completions/mean_length": 172.08222198486328,
|
||
|
|
"eval_completions/mean_terminated_length": 172.08222198486328,
|
||
|
|
"eval_completions/min_length": 93.0,
|
||
|
|
"eval_completions/min_terminated_length": 93.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 664628418.0,
|
||
|
|
"eval_reward": 1.0765975713729858,
|
||
|
|
"eval_reward_std": 0.27346640825271606,
|
||
|
|
"eval_rewards/accuracy_reward": 0.375,
|
||
|
|
"eval_rewards/brier_reward": 0.7840966582298279,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4926757961511612,
|
||
|
|
"eval_rewards/smce": -0.11805175989866257,
|
||
|
|
"eval_runtime": 11.8017,
|
||
|
|
"eval_samples_per_second": 42.367,
|
||
|
|
"eval_steps_per_second": 0.169,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 937.0,
|
||
|
|
"completions/max_terminated_length": 524.4,
|
||
|
|
"completions/mean_length": 166.0533203125,
|
||
|
|
"completions/mean_terminated_length": 165.78526916503907,
|
||
|
|
"completions/min_length": 77.0,
|
||
|
|
"completions/min_terminated_length": 77.0,
|
||
|
|
"epoch": 0.656,
|
||
|
|
"grad_norm": 0.0011595977703109384,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 681185348.0,
|
||
|
|
"reward": 1.095522141456604,
|
||
|
|
"reward_std": 0.1261553093791008,
|
||
|
|
"rewards/accuracy_reward": 0.41953125,
|
||
|
|
"rewards/brier_reward": 0.7763510584831238,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5122119128704071,
|
||
|
|
"rewards/smce": -0.09287466555833816,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 926.4,
|
||
|
|
"completions/max_terminated_length": 485.8,
|
||
|
|
"completions/mean_length": 161.9060546875,
|
||
|
|
"completions/mean_terminated_length": 161.50301208496094,
|
||
|
|
"completions/min_length": 75.0,
|
||
|
|
"completions/min_terminated_length": 75.0,
|
||
|
|
"epoch": 0.672,
|
||
|
|
"grad_norm": 0.0007186342845670879,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 697756706.0,
|
||
|
|
"reward": 1.1261529922485352,
|
||
|
|
"reward_std": 0.09019538462162018,
|
||
|
|
"rewards/accuracy_reward": 0.456640625,
|
||
|
|
"rewards/brier_reward": 0.7979784965515136,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4447070300579071,
|
||
|
|
"rewards/smce": -0.040419237315654756,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 696.0,
|
||
|
|
"completions/max_terminated_length": 456.6,
|
||
|
|
"completions/mean_length": 159.845703125,
|
||
|
|
"completions/mean_terminated_length": 159.7115234375,
|
||
|
|
"completions/min_length": 78.4,
|
||
|
|
"completions/min_terminated_length": 78.4,
|
||
|
|
"epoch": 0.688,
|
||
|
|
"grad_norm": 0.0008274798165075481,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 714347446.0,
|
||
|
|
"reward": 1.1239240884780883,
|
||
|
|
"reward_std": 0.08857346624135971,
|
||
|
|
"rewards/accuracy_reward": 0.45234375,
|
||
|
|
"rewards/brier_reward": 0.7981483459472656,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4173779249191284,
|
||
|
|
"rewards/smce": -0.05094240251928568,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 460.4,
|
||
|
|
"completions/max_terminated_length": 460.4,
|
||
|
|
"completions/mean_length": 161.32236328125,
|
||
|
|
"completions/mean_terminated_length": 161.32236328125,
|
||
|
|
"completions/min_length": 78.4,
|
||
|
|
"completions/min_terminated_length": 78.4,
|
||
|
|
"epoch": 0.704,
|
||
|
|
"grad_norm": 0.0009050744120031595,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 730865531.0,
|
||
|
|
"reward": 1.1346723556518554,
|
||
|
|
"reward_std": 0.0825655698776245,
|
||
|
|
"rewards/accuracy_reward": 0.48037109375,
|
||
|
|
"rewards/brier_reward": 0.7921864032745362,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.41924610137939455,
|
||
|
|
"rewards/smce": -0.06231907419860363,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 635.0,
|
||
|
|
"completions/max_terminated_length": 418.2,
|
||
|
|
"completions/mean_length": 161.794921875,
|
||
|
|
"completions/mean_terminated_length": 161.6607177734375,
|
||
|
|
"completions/min_length": 77.8,
|
||
|
|
"completions/min_terminated_length": 77.8,
|
||
|
|
"epoch": 0.72,
|
||
|
|
"grad_norm": 0.0010375964920967817,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 747532167.0,
|
||
|
|
"reward": 1.1471279382705688,
|
||
|
|
"reward_std": 0.09896319657564163,
|
||
|
|
"rewards/accuracy_reward": 0.48896484375,
|
||
|
|
"rewards/brier_reward": 0.8072811841964722,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.48550487756729127,
|
||
|
|
"rewards/smce": -0.03786895275115967,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 639.8,
|
||
|
|
"completions/max_terminated_length": 414.8,
|
||
|
|
"completions/mean_length": 162.24970703125,
|
||
|
|
"completions/mean_terminated_length": 162.1154052734375,
|
||
|
|
"completions/min_length": 80.2,
|
||
|
|
"completions/min_terminated_length": 80.2,
|
||
|
|
"epoch": 0.736,
|
||
|
|
"grad_norm": 0.0011135365348309278,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 764133188.0,
|
||
|
|
"reward": 1.1404980421066284,
|
||
|
|
"reward_std": 0.11243823915719986,
|
||
|
|
"rewards/accuracy_reward": 0.49609375,
|
||
|
|
"rewards/brier_reward": 0.7878056287765502,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.5388007879257202,
|
||
|
|
"rewards/smce": -0.056135011836886406,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 422.2,
|
||
|
|
"completions/max_terminated_length": 422.2,
|
||
|
|
"completions/mean_length": 163.6078125,
|
||
|
|
"completions/mean_terminated_length": 163.6078125,
|
||
|
|
"completions/min_length": 75.4,
|
||
|
|
"completions/min_terminated_length": 75.4,
|
||
|
|
"epoch": 0.752,
|
||
|
|
"grad_norm": 0.0010154708288609982,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0001,
|
||
|
|
"num_tokens": 781035732.0,
|
||
|
|
"reward": 1.1349368810653686,
|
||
|
|
"reward_std": 0.1062909796833992,
|
||
|
|
"rewards/accuracy_reward": 0.47587890625,
|
||
|
|
"rewards/brier_reward": 0.7961811423301697,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4935058653354645,
|
||
|
|
"rewards/smce": -0.04374487325549126,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 416.6,
|
||
|
|
"completions/max_terminated_length": 416.6,
|
||
|
|
"completions/mean_length": 169.75712890625,
|
||
|
|
"completions/mean_terminated_length": 169.75712890625,
|
||
|
|
"completions/min_length": 79.6,
|
||
|
|
"completions/min_terminated_length": 79.6,
|
||
|
|
"epoch": 0.768,
|
||
|
|
"grad_norm": 0.0009220530628226697,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 797706749.0,
|
||
|
|
"reward": 1.1410370349884034,
|
||
|
|
"reward_std": 0.09544525891542435,
|
||
|
|
"rewards/accuracy_reward": 0.47705078125,
|
||
|
|
"rewards/brier_reward": 0.8060692191123963,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4754668116569519,
|
||
|
|
"rewards/smce": -0.018984863162040712,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 428.4,
|
||
|
|
"completions/max_terminated_length": 428.4,
|
||
|
|
"completions/mean_length": 171.64931640625,
|
||
|
|
"completions/mean_terminated_length": 171.64931640625,
|
||
|
|
"completions/min_length": 84.4,
|
||
|
|
"completions/min_terminated_length": 84.4,
|
||
|
|
"epoch": 0.784,
|
||
|
|
"grad_norm": 0.0008822088711895049,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 814638806.0,
|
||
|
|
"reward": 1.1337279319763183,
|
||
|
|
"reward_std": 0.10449115782976151,
|
||
|
|
"rewards/accuracy_reward": 0.4818359375,
|
||
|
|
"rewards/brier_reward": 0.7891198754310608,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.48597265481948854,
|
||
|
|
"rewards/smce": -0.0700188372284174,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1085.4,
|
||
|
|
"completions/max_terminated_length": 453.6,
|
||
|
|
"completions/mean_length": 170.1076171875,
|
||
|
|
"completions/mean_terminated_length": 169.7075622558594,
|
||
|
|
"completions/min_length": 85.2,
|
||
|
|
"completions/min_terminated_length": 85.2,
|
||
|
|
"epoch": 0.8,
|
||
|
|
"grad_norm": 0.0010923626832664013,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 831391268.0,
|
||
|
|
"reward": 1.151297688484192,
|
||
|
|
"reward_std": 0.11599037945270538,
|
||
|
|
"rewards/accuracy_reward": 0.5,
|
||
|
|
"rewards/brier_reward": 0.805100429058075,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.523035180568695,
|
||
|
|
"rewards/smce": -0.04035660531371832,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 318.0,
|
||
|
|
"eval_completions/max_terminated_length": 318.0,
|
||
|
|
"eval_completions/mean_length": 170.98802185058594,
|
||
|
|
"eval_completions/mean_terminated_length": 170.98802185058594,
|
||
|
|
"eval_completions/min_length": 88.5,
|
||
|
|
"eval_completions/min_terminated_length": 88.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 831391268.0,
|
||
|
|
"eval_reward": 1.0831419825553894,
|
||
|
|
"eval_reward_std": 0.28824666142463684,
|
||
|
|
"eval_rewards/accuracy_reward": 0.375,
|
||
|
|
"eval_rewards/brier_reward": 0.7978320419788361,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.5054687708616257,
|
||
|
|
"eval_rewards/smce": -0.1309814490377903,
|
||
|
|
"eval_runtime": 11.3528,
|
||
|
|
"eval_samples_per_second": 44.042,
|
||
|
|
"eval_steps_per_second": 0.176,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1094.8,
|
||
|
|
"completions/max_terminated_length": 434.6,
|
||
|
|
"completions/mean_length": 169.755078125,
|
||
|
|
"completions/mean_terminated_length": 169.3546356201172,
|
||
|
|
"completions/min_length": 83.6,
|
||
|
|
"completions/min_terminated_length": 83.6,
|
||
|
|
"epoch": 0.816,
|
||
|
|
"grad_norm": 0.0011063864221796393,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 848228728.0,
|
||
|
|
"reward": 1.1495605945587157,
|
||
|
|
"reward_std": 0.11100535690784455,
|
||
|
|
"rewards/accuracy_reward": 0.49404296875,
|
||
|
|
"rewards/brier_reward": 0.806819224357605,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.5094443440437317,
|
||
|
|
"rewards/smce": -0.02897880058735609,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 651.6,
|
||
|
|
"completions/max_terminated_length": 435.2,
|
||
|
|
"completions/mean_length": 176.95146484375,
|
||
|
|
"completions/mean_terminated_length": 176.8187255859375,
|
||
|
|
"completions/min_length": 87.4,
|
||
|
|
"completions/min_terminated_length": 87.4,
|
||
|
|
"epoch": 0.832,
|
||
|
|
"grad_norm": 0.001013425295241177,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 865049063.0,
|
||
|
|
"reward": 1.1530806064605712,
|
||
|
|
"reward_std": 0.09840368032455445,
|
||
|
|
"rewards/accuracy_reward": 0.4978515625,
|
||
|
|
"rewards/brier_reward": 0.8105565905570984,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.463302743434906,
|
||
|
|
"rewards/smce": -0.04300446677953005,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 676.0,
|
||
|
|
"completions/max_terminated_length": 439.2,
|
||
|
|
"completions/mean_length": 172.416015625,
|
||
|
|
"completions/mean_terminated_length": 172.28269958496094,
|
||
|
|
"completions/min_length": 86.4,
|
||
|
|
"completions/min_terminated_length": 86.4,
|
||
|
|
"epoch": 0.848,
|
||
|
|
"grad_norm": 0.0009189588017761707,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 881828971.0,
|
||
|
|
"reward": 1.1401946306228639,
|
||
|
|
"reward_std": 0.11134238541126251,
|
||
|
|
"rewards/accuracy_reward": 0.47490234375,
|
||
|
|
"rewards/brier_reward": 0.8072995543479919,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4904062569141388,
|
||
|
|
"rewards/smce": -0.034317873790860175,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 881.4,
|
||
|
|
"completions/max_terminated_length": 436.8,
|
||
|
|
"completions/mean_length": 174.81015625,
|
||
|
|
"completions/mean_terminated_length": 174.2799499511719,
|
||
|
|
"completions/min_length": 81.6,
|
||
|
|
"completions/min_terminated_length": 81.6,
|
||
|
|
"epoch": 0.864,
|
||
|
|
"grad_norm": 0.0008723426144570112,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 898605843.0,
|
||
|
|
"reward": 1.1487454652786255,
|
||
|
|
"reward_std": 0.10216106027364731,
|
||
|
|
"rewards/accuracy_reward": 0.5072265625,
|
||
|
|
"rewards/brier_reward": 0.79281005859375,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.49146581888198854,
|
||
|
|
"rewards/smce": -0.04311874024569988,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 694.2,
|
||
|
|
"completions/max_terminated_length": 498.8,
|
||
|
|
"completions/mean_length": 179.809375,
|
||
|
|
"completions/mean_terminated_length": 179.677294921875,
|
||
|
|
"completions/min_length": 93.6,
|
||
|
|
"completions/min_terminated_length": 93.6,
|
||
|
|
"epoch": 0.88,
|
||
|
|
"grad_norm": 0.0009168738033622503,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 915594163.0,
|
||
|
|
"reward": 1.1366500377655029,
|
||
|
|
"reward_std": 0.08010695576667785,
|
||
|
|
"rewards/accuracy_reward": 0.462890625,
|
||
|
|
"rewards/brier_reward": 0.8138427495956421,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.41804786324501036,
|
||
|
|
"rewards/smce": -0.0667283084243536,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 736.6,
|
||
|
|
"completions/max_terminated_length": 637.4,
|
||
|
|
"completions/mean_length": 185.1380859375,
|
||
|
|
"completions/mean_terminated_length": 185.00626220703126,
|
||
|
|
"completions/min_length": 94.0,
|
||
|
|
"completions/min_terminated_length": 94.0,
|
||
|
|
"epoch": 0.896,
|
||
|
|
"grad_norm": 0.0009079832234419882,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 932600825.0,
|
||
|
|
"reward": 1.1398839712142945,
|
||
|
|
"reward_std": 0.09819626659154893,
|
||
|
|
"rewards/accuracy_reward": 0.47734375,
|
||
|
|
"rewards/brier_reward": 0.8047329187393188,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.47663772106170654,
|
||
|
|
"rewards/smce": -0.04424012675881386,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 973.0,
|
||
|
|
"completions/max_terminated_length": 515.2,
|
||
|
|
"completions/mean_length": 186.6216796875,
|
||
|
|
"completions/mean_terminated_length": 186.35760803222655,
|
||
|
|
"completions/min_length": 93.6,
|
||
|
|
"completions/min_terminated_length": 93.6,
|
||
|
|
"epoch": 0.912,
|
||
|
|
"grad_norm": 0.0010032489662989974,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 949563127.0,
|
||
|
|
"reward": 1.1336450338363648,
|
||
|
|
"reward_std": 0.11059280037879944,
|
||
|
|
"rewards/accuracy_reward": 0.47021484375,
|
||
|
|
"rewards/brier_reward": 0.8010426044464112,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5327714800834655,
|
||
|
|
"rewards/smce": -0.07546240240335464,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 521.8,
|
||
|
|
"completions/max_terminated_length": 521.8,
|
||
|
|
"completions/mean_length": 178.80126953125,
|
||
|
|
"completions/mean_terminated_length": 178.80126953125,
|
||
|
|
"completions/min_length": 94.2,
|
||
|
|
"completions/min_terminated_length": 94.2,
|
||
|
|
"epoch": 0.928,
|
||
|
|
"grad_norm": 0.0009838527766987681,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 966420868.0,
|
||
|
|
"reward": 1.1457917451858521,
|
||
|
|
"reward_std": 0.09142312407493591,
|
||
|
|
"rewards/accuracy_reward": 0.4966796875,
|
||
|
|
"rewards/brier_reward": 0.7967879176139832,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.49010254740715026,
|
||
|
|
"rewards/smce": -0.037700850144028665,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 630.2,
|
||
|
|
"completions/max_terminated_length": 428.4,
|
||
|
|
"completions/mean_length": 169.31669921875,
|
||
|
|
"completions/mean_terminated_length": 169.18387145996093,
|
||
|
|
"completions/min_length": 86.0,
|
||
|
|
"completions/min_terminated_length": 86.0,
|
||
|
|
"epoch": 0.944,
|
||
|
|
"grad_norm": 0.000964009203016758,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 983130095.0,
|
||
|
|
"reward": 1.1439347505569457,
|
||
|
|
"reward_std": 0.09324800819158555,
|
||
|
|
"rewards/accuracy_reward": 0.494921875,
|
||
|
|
"rewards/brier_reward": 0.7968281626701355,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4685791075229645,
|
||
|
|
"rewards/smce": -0.07567493021488189,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 664.6,
|
||
|
|
"completions/max_terminated_length": 436.0,
|
||
|
|
"completions/mean_length": 163.23974609375,
|
||
|
|
"completions/mean_terminated_length": 163.10575256347656,
|
||
|
|
"completions/min_length": 83.4,
|
||
|
|
"completions/min_terminated_length": 83.4,
|
||
|
|
"epoch": 0.96,
|
||
|
|
"grad_norm": 0.001006408827379346,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 999741990.0,
|
||
|
|
"reward": 1.1354127168655395,
|
||
|
|
"reward_std": 0.08625964224338531,
|
||
|
|
"rewards/accuracy_reward": 0.4607421875,
|
||
|
|
"rewards/brier_reward": 0.8117612481117249,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4783691465854645,
|
||
|
|
"rewards/smce": -0.03162573650479317,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.96,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 319.0,
|
||
|
|
"eval_completions/max_terminated_length": 319.0,
|
||
|
|
"eval_completions/mean_length": 158.7119598388672,
|
||
|
|
"eval_completions/mean_terminated_length": 158.7119598388672,
|
||
|
|
"eval_completions/min_length": 101.0,
|
||
|
|
"eval_completions/min_terminated_length": 101.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 999741990.0,
|
||
|
|
"eval_reward": 1.0915340185165405,
|
||
|
|
"eval_reward_std": 0.27021801471710205,
|
||
|
|
"eval_rewards/accuracy_reward": 0.384765625,
|
||
|
|
"eval_rewards/brier_reward": 0.8024950921535492,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4678710997104645,
|
||
|
|
"eval_rewards/smce": -0.08387207612395287,
|
||
|
|
"eval_runtime": 11.1096,
|
||
|
|
"eval_samples_per_second": 45.006,
|
||
|
|
"eval_steps_per_second": 0.18,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 877.2,
|
||
|
|
"completions/max_terminated_length": 459.0,
|
||
|
|
"completions/mean_length": 156.80107421875,
|
||
|
|
"completions/mean_terminated_length": 156.53237609863282,
|
||
|
|
"completions/min_length": 80.6,
|
||
|
|
"completions/min_terminated_length": 80.6,
|
||
|
|
"epoch": 0.976,
|
||
|
|
"grad_norm": 0.000879038474522531,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 1016208753.0,
|
||
|
|
"reward": 1.1549882173538208,
|
||
|
|
"reward_std": 0.08301678597927094,
|
||
|
|
"rewards/accuracy_reward": 0.49765625,
|
||
|
|
"rewards/brier_reward": 0.8142663717269898,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4712851524353027,
|
||
|
|
"rewards/smce": -0.03503466323018074,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 880.0,
|
||
|
|
"completions/max_terminated_length": 482.8,
|
||
|
|
"completions/mean_length": 152.35126953125,
|
||
|
|
"completions/mean_terminated_length": 152.08108215332032,
|
||
|
|
"completions/min_length": 75.8,
|
||
|
|
"completions/min_terminated_length": 75.8,
|
||
|
|
"epoch": 0.992,
|
||
|
|
"grad_norm": 0.0010177699150517583,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 1032897310.0,
|
||
|
|
"reward": 1.140252137184143,
|
||
|
|
"reward_std": 0.0865005910396576,
|
||
|
|
"rewards/accuracy_reward": 0.48935546875,
|
||
|
|
"rewards/brier_reward": 0.7936301112174988,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.44715137481689454,
|
||
|
|
"rewards/smce": -0.043783336877822876,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 382.5,
|
||
|
|
"completions/max_terminated_length": 382.5,
|
||
|
|
"completions/mean_length": 155.24308013916016,
|
||
|
|
"completions/mean_terminated_length": 155.24308013916016,
|
||
|
|
"completions/min_length": 78.5,
|
||
|
|
"completions/min_terminated_length": 78.5,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"num_tokens": 1039540391.0,
|
||
|
|
"reward": 1.1383739113807678,
|
||
|
|
"reward_std": 0.09339407458901405,
|
||
|
|
"rewards/accuracy_reward": 0.4765625,
|
||
|
|
"rewards/brier_reward": 0.8023821711540222,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.49127197265625,
|
||
|
|
"rewards/smce": -0.043958742171525955,
|
||
|
|
"step": 312,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.003989085646259273,
|
||
|
|
"train_runtime": 42377.2177,
|
||
|
|
"train_samples_per_second": 0.472,
|
||
|
|
"train_steps_per_second": 0.007
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 312,
|
||
|
|
"num_input_tokens_seen": 1039540391,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 60,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 4,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|