1944 lines
72 KiB
JSON
1944 lines
72 KiB
JSON
|
{
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"eval_steps": 50,
|
||
|
|
"global_step": 312,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.1029296875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1533.0,
|
||
|
|
"completions/mean_length": 463.92587890625,
|
||
|
|
"completions/mean_terminated_length": 340.91392822265624,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.016,
|
||
|
|
"grad_norm": 0.003800533711910248,
|
||
|
|
"learning_rate": 3.1249999999999997e-07,
|
||
|
|
"loss": 0.0456,
|
||
|
|
"num_tokens": 19594633.0,
|
||
|
|
"reward": 0.21571238338947296,
|
||
|
|
"reward_std": 0.3878848910331726,
|
||
|
|
"rewards/accuracy_reward": 0.06201171875,
|
||
|
|
"rewards/brier_reward": 0.10856110006570815,
|
||
|
|
"rewards/confidence_one_or_zero": 0.413671875,
|
||
|
|
"rewards/format_reward": 0.261328125,
|
||
|
|
"rewards/mean_confidence_reward": 0.5398741960525513,
|
||
|
|
"rewards/smce": -0.4780743718147278,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.10458984375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1534.0,
|
||
|
|
"completions/mean_length": 445.432421875,
|
||
|
|
"completions/mean_terminated_length": 318.1658874511719,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.032,
|
||
|
|
"grad_norm": 0.004327881615608931,
|
||
|
|
"learning_rate": 6.249999999999999e-07,
|
||
|
|
"loss": 0.052,
|
||
|
|
"num_tokens": 39256213.0,
|
||
|
|
"reward": 0.22842271625995636,
|
||
|
|
"reward_std": 0.3874061942100525,
|
||
|
|
"rewards/accuracy_reward": 0.0625,
|
||
|
|
"rewards/brier_reward": 0.1117446094751358,
|
||
|
|
"rewards/confidence_one_or_zero": 0.432421875,
|
||
|
|
"rewards/format_reward": 0.28310546875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5687525153160096,
|
||
|
|
"rewards/smce": -0.506620317697525,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.050390625,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1472.8,
|
||
|
|
"completions/mean_length": 305.22607421875,
|
||
|
|
"completions/mean_terminated_length": 240.37628784179688,
|
||
|
|
"completions/min_length": 1.4,
|
||
|
|
"completions/min_terminated_length": 1.4,
|
||
|
|
"epoch": 0.048,
|
||
|
|
"grad_norm": 0.004292675759643316,
|
||
|
|
"learning_rate": 9.374999999999999e-07,
|
||
|
|
"loss": 0.0542,
|
||
|
|
"num_tokens": 57430464.0,
|
||
|
|
"reward": 0.4086169540882111,
|
||
|
|
"reward_std": 0.43530423045158384,
|
||
|
|
"rewards/accuracy_reward": 0.1064453125,
|
||
|
|
"rewards/brier_reward": 0.20014296174049379,
|
||
|
|
"rewards/confidence_one_or_zero": 0.537109375,
|
||
|
|
"rewards/format_reward": 0.51123046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.6928913235664368,
|
||
|
|
"rewards/smce": -0.5873086810111999,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.01923828125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1438.8,
|
||
|
|
"completions/mean_length": 205.31982421875,
|
||
|
|
"completions/mean_terminated_length": 179.24837951660157,
|
||
|
|
"completions/min_length": 10.8,
|
||
|
|
"completions/min_terminated_length": 10.8,
|
||
|
|
"epoch": 0.064,
|
||
|
|
"grad_norm": 0.0027189909014850855,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0359,
|
||
|
|
"num_tokens": 74451339.0,
|
||
|
|
"reward": 0.6751530885696411,
|
||
|
|
"reward_std": 0.39800527691841125,
|
||
|
|
"rewards/accuracy_reward": 0.18740234375,
|
||
|
|
"rewards/brier_reward": 0.3311912059783936,
|
||
|
|
"rewards/confidence_one_or_zero": 0.5701171875,
|
||
|
|
"rewards/format_reward": 0.83232421875,
|
||
|
|
"rewards/mean_confidence_reward": 0.7987796306610108,
|
||
|
|
"rewards/smce": -0.614277696609497,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.01142578125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1298.4,
|
||
|
|
"completions/mean_length": 157.06796875,
|
||
|
|
"completions/mean_terminated_length": 141.12780151367187,
|
||
|
|
"completions/min_length": 23.0,
|
||
|
|
"completions/min_terminated_length": 23.0,
|
||
|
|
"epoch": 0.08,
|
||
|
|
"grad_norm": 0.0027302594389766455,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0167,
|
||
|
|
"num_tokens": 90992867.0,
|
||
|
|
"reward": 0.7987664818763733,
|
||
|
|
"reward_std": 0.3233607649803162,
|
||
|
|
"rewards/accuracy_reward": 0.2138671875,
|
||
|
|
"rewards/brier_reward": 0.41909300684928896,
|
||
|
|
"rewards/confidence_one_or_zero": 0.49765625,
|
||
|
|
"rewards/format_reward": 0.96513671875,
|
||
|
|
"rewards/mean_confidence_reward": 0.7765803933143616,
|
||
|
|
"rewards/smce": -0.5664987325668335,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00634765625,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1184.8,
|
||
|
|
"completions/mean_length": 135.8953125,
|
||
|
|
"completions/mean_terminated_length": 126.9502456665039,
|
||
|
|
"completions/min_length": 27.8,
|
||
|
|
"completions/min_terminated_length": 27.8,
|
||
|
|
"epoch": 0.096,
|
||
|
|
"grad_norm": 0.002199189504608512,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0104,
|
||
|
|
"num_tokens": 107429043.0,
|
||
|
|
"reward": 0.8861237287521362,
|
||
|
|
"reward_std": 0.2804286539554596,
|
||
|
|
"rewards/accuracy_reward": 0.23486328125,
|
||
|
|
"rewards/brier_reward": 0.5540830910205841,
|
||
|
|
"rewards/confidence_one_or_zero": 0.3076171875,
|
||
|
|
"rewards/format_reward": 0.98369140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.6132706642150879,
|
||
|
|
"rewards/smce": -0.39209535121917727,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.003515625,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1061.6,
|
||
|
|
"completions/mean_length": 121.37578125,
|
||
|
|
"completions/mean_terminated_length": 116.38601531982422,
|
||
|
|
"completions/min_length": 36.2,
|
||
|
|
"completions/min_terminated_length": 36.2,
|
||
|
|
"epoch": 0.112,
|
||
|
|
"grad_norm": 0.0014616711996495724,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0069,
|
||
|
|
"num_tokens": 123781403.0,
|
||
|
|
"reward": 0.989423644542694,
|
||
|
|
"reward_std": 0.19730665683746337,
|
||
|
|
"rewards/accuracy_reward": 0.26953125,
|
||
|
|
"rewards/brier_reward": 0.7158191919326782,
|
||
|
|
"rewards/confidence_one_or_zero": 0.18125,
|
||
|
|
"rewards/format_reward": 0.99365234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.38006125688552855,
|
||
|
|
"rewards/smce": -0.15658431351184846,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0021484375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 876.8,
|
||
|
|
"completions/mean_length": 115.8734375,
|
||
|
|
"completions/mean_terminated_length": 112.8156021118164,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"epoch": 0.128,
|
||
|
|
"grad_norm": 0.0011532431235536933,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0045,
|
||
|
|
"num_tokens": 139884619.0,
|
||
|
|
"reward": 1.0200982809066772,
|
||
|
|
"reward_std": 0.13898254334926605,
|
||
|
|
"rewards/accuracy_reward": 0.27294921875,
|
||
|
|
"rewards/brier_reward": 0.7711090922355652,
|
||
|
|
"rewards/confidence_one_or_zero": 0.1431640625,
|
||
|
|
"rewards/format_reward": 0.99619140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.263618403673172,
|
||
|
|
"rewards/smce": -0.053969383239746094,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00078125,
|
||
|
|
"completions/max_length": 1296.2,
|
||
|
|
"completions/max_terminated_length": 526.0,
|
||
|
|
"completions/mean_length": 111.2216796875,
|
||
|
|
"completions/mean_terminated_length": 110.10794219970703,
|
||
|
|
"completions/min_length": 45.0,
|
||
|
|
"completions/min_terminated_length": 45.0,
|
||
|
|
"epoch": 0.144,
|
||
|
|
"grad_norm": 0.0008673432166688144,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.002,
|
||
|
|
"num_tokens": 155973961.0,
|
||
|
|
"reward": 1.0399757146835327,
|
||
|
|
"reward_std": 0.1108901247382164,
|
||
|
|
"rewards/accuracy_reward": 0.32666015625,
|
||
|
|
"rewards/brier_reward": 0.7554583191871643,
|
||
|
|
"rewards/confidence_one_or_zero": 0.09560546875,
|
||
|
|
"rewards/format_reward": 0.99794921875,
|
||
|
|
"rewards/mean_confidence_reward": 0.21799394190311433,
|
||
|
|
"rewards/smce": -0.11687580943107605,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0005859375,
|
||
|
|
"completions/max_length": 1087.6,
|
||
|
|
"completions/max_terminated_length": 440.8,
|
||
|
|
"completions/mean_length": 112.95947265625,
|
||
|
|
"completions/mean_terminated_length": 112.12568817138671,
|
||
|
|
"completions/min_length": 50.6,
|
||
|
|
"completions/min_terminated_length": 50.6,
|
||
|
|
"epoch": 0.16,
|
||
|
|
"grad_norm": 0.0007697119144722819,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"num_tokens": 172151594.0,
|
||
|
|
"reward": 1.041567873954773,
|
||
|
|
"reward_std": 0.09320179969072342,
|
||
|
|
"rewards/accuracy_reward": 0.32041015625,
|
||
|
|
"rewards/brier_reward": 0.764100170135498,
|
||
|
|
"rewards/confidence_one_or_zero": 0.05087890625,
|
||
|
|
"rewards/format_reward": 0.99873046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.21918963193893432,
|
||
|
|
"rewards/smce": -0.10564200878143311,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 332.5,
|
||
|
|
"eval_completions/max_terminated_length": 332.5,
|
||
|
|
"eval_completions/mean_length": 114.48052978515625,
|
||
|
|
"eval_completions/mean_terminated_length": 114.48052978515625,
|
||
|
|
"eval_completions/min_length": 60.0,
|
||
|
|
"eval_completions/min_terminated_length": 60.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 172151594.0,
|
||
|
|
"eval_reward": 1.0331834554672241,
|
||
|
|
"eval_reward_std": 0.125154048204422,
|
||
|
|
"eval_rewards/accuracy_reward": 0.267578125,
|
||
|
|
"eval_rewards/brier_reward": 0.7988348007202148,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.037109375,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.22960937768220901,
|
||
|
|
"eval_rewards/smce": -0.04650058038532734,
|
||
|
|
"eval_runtime": 12.04,
|
||
|
|
"eval_samples_per_second": 41.528,
|
||
|
|
"eval_steps_per_second": 0.166,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00068359375,
|
||
|
|
"completions/max_length": 1417.4,
|
||
|
|
"completions/max_terminated_length": 519.0,
|
||
|
|
"completions/mean_length": 113.7537109375,
|
||
|
|
"completions/mean_terminated_length": 112.78092041015626,
|
||
|
|
"completions/min_length": 49.2,
|
||
|
|
"completions/min_terminated_length": 49.2,
|
||
|
|
"epoch": 0.176,
|
||
|
|
"grad_norm": 0.0008332604193128645,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"num_tokens": 188553552.0,
|
||
|
|
"reward": 1.0444111824035645,
|
||
|
|
"reward_std": 0.093848617374897,
|
||
|
|
"rewards/accuracy_reward": 0.3125,
|
||
|
|
"rewards/brier_reward": 0.7778581500053405,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0224609375,
|
||
|
|
"rewards/format_reward": 0.99853515625,
|
||
|
|
"rewards/mean_confidence_reward": 0.24422344267368318,
|
||
|
|
"rewards/smce": -0.0715064637362957,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 855.2,
|
||
|
|
"completions/max_terminated_length": 396.8,
|
||
|
|
"completions/mean_length": 114.6798828125,
|
||
|
|
"completions/mean_terminated_length": 114.40217590332031,
|
||
|
|
"completions/min_length": 53.8,
|
||
|
|
"completions/min_terminated_length": 53.8,
|
||
|
|
"epoch": 0.192,
|
||
|
|
"grad_norm": 0.0008707884699106216,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 204542690.0,
|
||
|
|
"reward": 1.062576985359192,
|
||
|
|
"reward_std": 0.09943905770778656,
|
||
|
|
"rewards/accuracy_reward": 0.353515625,
|
||
|
|
"rewards/brier_reward": 0.7720983743667602,
|
||
|
|
"rewards/confidence_one_or_zero": 0.008203125,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.29078262448310854,
|
||
|
|
"rewards/smce": -0.0698842965066433,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1290.6,
|
||
|
|
"completions/max_terminated_length": 299.4,
|
||
|
|
"completions/mean_length": 121.6306640625,
|
||
|
|
"completions/mean_terminated_length": 120.94003753662109,
|
||
|
|
"completions/min_length": 47.6,
|
||
|
|
"completions/min_terminated_length": 47.6,
|
||
|
|
"epoch": 0.208,
|
||
|
|
"grad_norm": 0.0007853489951230586,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 220820412.0,
|
||
|
|
"reward": 1.0697309255599976,
|
||
|
|
"reward_std": 0.11286971271038056,
|
||
|
|
"rewards/accuracy_reward": 0.3732421875,
|
||
|
|
"rewards/brier_reward": 0.7671383023262024,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00322265625,
|
||
|
|
"rewards/format_reward": 0.99912109375,
|
||
|
|
"rewards/mean_confidence_reward": 0.3380278408527374,
|
||
|
|
"rewards/smce": -0.04038693383336067,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 824.0,
|
||
|
|
"completions/max_terminated_length": 399.2,
|
||
|
|
"completions/mean_length": 123.864453125,
|
||
|
|
"completions/mean_terminated_length": 123.31305847167968,
|
||
|
|
"completions/min_length": 59.2,
|
||
|
|
"completions/min_terminated_length": 59.2,
|
||
|
|
"epoch": 0.224,
|
||
|
|
"grad_norm": 0.0009797868551686406,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0016,
|
||
|
|
"num_tokens": 237241968.0,
|
||
|
|
"reward": 1.0696279525756835,
|
||
|
|
"reward_std": 0.11271759867668152,
|
||
|
|
"rewards/accuracy_reward": 0.36845703125,
|
||
|
|
"rewards/brier_reward": 0.7713197946548462,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00244140625,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.3636253893375397,
|
||
|
|
"rewards/smce": -0.033345287293195726,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 531.2,
|
||
|
|
"completions/max_terminated_length": 279.2,
|
||
|
|
"completions/mean_length": 122.9078125,
|
||
|
|
"completions/mean_terminated_length": 122.76952819824218,
|
||
|
|
"completions/min_length": 60.0,
|
||
|
|
"completions/min_terminated_length": 60.0,
|
||
|
|
"epoch": 0.24,
|
||
|
|
"grad_norm": 0.0009892035741358995,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 253752224.0,
|
||
|
|
"reward": 1.0851356983184814,
|
||
|
|
"reward_std": 0.12177922129631043,
|
||
|
|
"rewards/accuracy_reward": 0.40810546875,
|
||
|
|
"rewards/brier_reward": 0.7624287366867065,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0005859375,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.3889783203601837,
|
||
|
|
"rewards/smce": -0.06828636862337589,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 296.4,
|
||
|
|
"completions/max_terminated_length": 296.4,
|
||
|
|
"completions/mean_length": 119.69375,
|
||
|
|
"completions/mean_terminated_length": 119.69375,
|
||
|
|
"completions/min_length": 58.6,
|
||
|
|
"completions/min_terminated_length": 58.6,
|
||
|
|
"epoch": 0.256,
|
||
|
|
"grad_norm": 0.0009709701989777386,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0,
|
||
|
|
"num_tokens": 270032704.0,
|
||
|
|
"reward": 1.0745017528533936,
|
||
|
|
"reward_std": 0.11845734417438507,
|
||
|
|
"rewards/accuracy_reward": 0.3814453125,
|
||
|
|
"rewards/brier_reward": 0.7679124712944031,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0009765625,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4074843883514404,
|
||
|
|
"rewards/smce": -0.062050586566329,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 543.8,
|
||
|
|
"completions/max_terminated_length": 303.4,
|
||
|
|
"completions/mean_length": 121.2091796875,
|
||
|
|
"completions/mean_terminated_length": 121.07110595703125,
|
||
|
|
"completions/min_length": 58.6,
|
||
|
|
"completions/min_terminated_length": 58.6,
|
||
|
|
"epoch": 0.272,
|
||
|
|
"grad_norm": 0.0009048814536072314,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 286239582.0,
|
||
|
|
"reward": 1.0838090896606445,
|
||
|
|
"reward_std": 0.11522045135498046,
|
||
|
|
"rewards/accuracy_reward": 0.40185546875,
|
||
|
|
"rewards/brier_reward": 0.7661949157714844,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4112128973007202,
|
||
|
|
"rewards/smce": -0.04232981689274311,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 314.4,
|
||
|
|
"completions/max_terminated_length": 314.4,
|
||
|
|
"completions/mean_length": 119.34541015625,
|
||
|
|
"completions/mean_terminated_length": 119.34541015625,
|
||
|
|
"completions/min_length": 58.4,
|
||
|
|
"completions/min_terminated_length": 58.4,
|
||
|
|
"epoch": 0.288,
|
||
|
|
"grad_norm": 0.0011435942724347115,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 302419855.0,
|
||
|
|
"reward": 1.0826804637908936,
|
||
|
|
"reward_std": 0.114729243516922,
|
||
|
|
"rewards/accuracy_reward": 0.39580078125,
|
||
|
|
"rewards/brier_reward": 0.7699063777923584,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4135368287563324,
|
||
|
|
"rewards/smce": -0.05406544618308544,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 628.0,
|
||
|
|
"completions/max_terminated_length": 370.6,
|
||
|
|
"completions/mean_length": 119.10458984375,
|
||
|
|
"completions/mean_terminated_length": 118.96620635986328,
|
||
|
|
"completions/min_length": 58.6,
|
||
|
|
"completions/min_terminated_length": 58.6,
|
||
|
|
"epoch": 0.304,
|
||
|
|
"grad_norm": 0.0010342359310016036,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0,
|
||
|
|
"num_tokens": 318569438.0,
|
||
|
|
"reward": 1.0892314672470094,
|
||
|
|
"reward_std": 0.11390392035245896,
|
||
|
|
"rewards/accuracy_reward": 0.41328125,
|
||
|
|
"rewards/brier_reward": 0.7654102206230163,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.407882821559906,
|
||
|
|
"rewards/smce": -0.03404633477330208,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 535.6,
|
||
|
|
"completions/max_terminated_length": 319.8,
|
||
|
|
"completions/mean_length": 120.22197265625,
|
||
|
|
"completions/mean_terminated_length": 119.94470672607422,
|
||
|
|
"completions/min_length": 59.6,
|
||
|
|
"completions/min_terminated_length": 59.6,
|
||
|
|
"epoch": 0.32,
|
||
|
|
"grad_norm": 0.0009286344284191728,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 334889215.0,
|
||
|
|
"reward": 1.0949872255325317,
|
||
|
|
"reward_std": 0.10952811241149903,
|
||
|
|
"rewards/accuracy_reward": 0.419921875,
|
||
|
|
"rewards/brier_reward": 0.7704041719436645,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.41058760285377505,
|
||
|
|
"rewards/smce": -0.05937949083745479,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 223.0,
|
||
|
|
"eval_completions/max_terminated_length": 223.0,
|
||
|
|
"eval_completions/mean_length": 120.3595962524414,
|
||
|
|
"eval_completions/mean_terminated_length": 120.3595962524414,
|
||
|
|
"eval_completions/min_length": 66.5,
|
||
|
|
"eval_completions/min_terminated_length": 66.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 334889215.0,
|
||
|
|
"eval_reward": 1.0504685044288635,
|
||
|
|
"eval_reward_std": 0.2088107019662857,
|
||
|
|
"eval_rewards/accuracy_reward": 0.318359375,
|
||
|
|
"eval_rewards/brier_reward": 0.7826702892780304,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4114453196525574,
|
||
|
|
"eval_rewards/smce": -0.09349317103624344,
|
||
|
|
"eval_runtime": 9.4015,
|
||
|
|
"eval_samples_per_second": 53.183,
|
||
|
|
"eval_steps_per_second": 0.213,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 534.8,
|
||
|
|
"completions/max_terminated_length": 289.0,
|
||
|
|
"completions/mean_length": 117.00869140625,
|
||
|
|
"completions/mean_terminated_length": 116.86989135742188,
|
||
|
|
"completions/min_length": 57.6,
|
||
|
|
"completions/min_terminated_length": 57.6,
|
||
|
|
"epoch": 0.336,
|
||
|
|
"grad_norm": 0.0009752543410286307,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 350809816.0,
|
||
|
|
"reward": 1.0989506721496582,
|
||
|
|
"reward_std": 0.1054120883345604,
|
||
|
|
"rewards/accuracy_reward": 0.4306640625,
|
||
|
|
"rewards/brier_reward": 0.7674848198890686,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4077809572219849,
|
||
|
|
"rewards/smce": -0.05311233140528202,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 273.0,
|
||
|
|
"completions/max_terminated_length": 273.0,
|
||
|
|
"completions/mean_length": 118.835546875,
|
||
|
|
"completions/mean_terminated_length": 118.835546875,
|
||
|
|
"completions/min_length": 56.8,
|
||
|
|
"completions/min_terminated_length": 56.8,
|
||
|
|
"epoch": 0.352,
|
||
|
|
"grad_norm": 0.0008025058778002858,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 367287108.0,
|
||
|
|
"reward": 1.0761260509490966,
|
||
|
|
"reward_std": 0.09988621026277542,
|
||
|
|
"rewards/accuracy_reward": 0.37802734375,
|
||
|
|
"rewards/brier_reward": 0.7743457198143006,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.39769991040229796,
|
||
|
|
"rewards/smce": -0.02415503077208996,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 303.8,
|
||
|
|
"completions/max_terminated_length": 303.8,
|
||
|
|
"completions/mean_length": 124.12080078125,
|
||
|
|
"completions/mean_terminated_length": 124.12080078125,
|
||
|
|
"completions/min_length": 60.6,
|
||
|
|
"completions/min_terminated_length": 60.6,
|
||
|
|
"epoch": 0.368,
|
||
|
|
"grad_norm": 0.0008689153473824263,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0002,
|
||
|
|
"num_tokens": 383623577.0,
|
||
|
|
"reward": 1.0884521007537842,
|
||
|
|
"reward_std": 0.09868751466274261,
|
||
|
|
"rewards/accuracy_reward": 0.39931640625,
|
||
|
|
"rewards/brier_reward": 0.7777201294898987,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4137275397777557,
|
||
|
|
"rewards/smce": -0.03548511806875467,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 559.2,
|
||
|
|
"completions/max_terminated_length": 303.4,
|
||
|
|
"completions/mean_length": 130.14931640625,
|
||
|
|
"completions/mean_terminated_length": 130.01228942871094,
|
||
|
|
"completions/min_length": 61.8,
|
||
|
|
"completions/min_terminated_length": 61.8,
|
||
|
|
"epoch": 0.384,
|
||
|
|
"grad_norm": 0.0009342688135802746,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 399812818.0,
|
||
|
|
"reward": 1.1017422437667848,
|
||
|
|
"reward_std": 0.10270306766033173,
|
||
|
|
"rewards/accuracy_reward": 0.42939453125,
|
||
|
|
"rewards/brier_reward": 0.7743234157562255,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.41212044954299926,
|
||
|
|
"rewards/smce": -0.03892633318901062,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 838.0,
|
||
|
|
"completions/max_terminated_length": 341.0,
|
||
|
|
"completions/mean_length": 138.803515625,
|
||
|
|
"completions/mean_terminated_length": 138.5307159423828,
|
||
|
|
"completions/min_length": 57.0,
|
||
|
|
"completions/min_terminated_length": 57.0,
|
||
|
|
"epoch": 0.4,
|
||
|
|
"grad_norm": 0.0009262771927751601,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 416270614.0,
|
||
|
|
"reward": 1.0920633316040038,
|
||
|
|
"reward_std": 0.10999695956707001,
|
||
|
|
"rewards/accuracy_reward": 0.40185546875,
|
||
|
|
"rewards/brier_reward": 0.7826905965805053,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4225615203380585,
|
||
|
|
"rewards/smce": -0.02956448132172227,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1100.8,
|
||
|
|
"completions/max_terminated_length": 430.0,
|
||
|
|
"completions/mean_length": 149.11748046875,
|
||
|
|
"completions/mean_terminated_length": 148.43955993652344,
|
||
|
|
"completions/min_length": 70.6,
|
||
|
|
"completions/min_terminated_length": 70.6,
|
||
|
|
"epoch": 0.416,
|
||
|
|
"grad_norm": 0.0009191810968331993,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.002,
|
||
|
|
"num_tokens": 432678761.0,
|
||
|
|
"reward": 1.0989661931991577,
|
||
|
|
"reward_std": 0.11646136194467545,
|
||
|
|
"rewards/accuracy_reward": 0.421875,
|
||
|
|
"rewards/brier_reward": 0.7765630722045899,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.42575586438179014,
|
||
|
|
"rewards/smce": -0.018213643692433834,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1067.2,
|
||
|
|
"completions/max_terminated_length": 394.8,
|
||
|
|
"completions/mean_length": 152.19921875,
|
||
|
|
"completions/mean_terminated_length": 151.7940216064453,
|
||
|
|
"completions/min_length": 69.6,
|
||
|
|
"completions/min_terminated_length": 69.6,
|
||
|
|
"epoch": 0.432,
|
||
|
|
"grad_norm": 0.0008486987790092826,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 449251617.0,
|
||
|
|
"reward": 1.1061060428619385,
|
||
|
|
"reward_std": 0.10609066933393478,
|
||
|
|
"rewards/accuracy_reward": 0.43486328125,
|
||
|
|
"rewards/brier_reward": 0.777781867980957,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4303037166595459,
|
||
|
|
"rewards/smce": -0.04326645694673061,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 859.4,
|
||
|
|
"completions/max_terminated_length": 387.0,
|
||
|
|
"completions/mean_length": 152.56455078125,
|
||
|
|
"completions/mean_terminated_length": 152.29410705566406,
|
||
|
|
"completions/min_length": 70.6,
|
||
|
|
"completions/min_terminated_length": 70.6,
|
||
|
|
"epoch": 0.448,
|
||
|
|
"grad_norm": 0.0008342221262864769,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 465766678.0,
|
||
|
|
"reward": 1.101392149925232,
|
||
|
|
"reward_std": 0.1103613555431366,
|
||
|
|
"rewards/accuracy_reward": 0.425,
|
||
|
|
"rewards/brier_reward": 0.778016197681427,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4370110690593719,
|
||
|
|
"rewards/smce": -0.037433745712041853,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 847.6,
|
||
|
|
"completions/max_terminated_length": 389.0,
|
||
|
|
"completions/mean_length": 154.2365234375,
|
||
|
|
"completions/mean_terminated_length": 153.96669311523436,
|
||
|
|
"completions/min_length": 67.6,
|
||
|
|
"completions/min_terminated_length": 67.6,
|
||
|
|
"epoch": 0.464,
|
||
|
|
"grad_norm": 0.0007826949004083872,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0014,
|
||
|
|
"num_tokens": 482516876.0,
|
||
|
|
"reward": 1.0809687376022339,
|
||
|
|
"reward_std": 0.10724099725484848,
|
||
|
|
"rewards/accuracy_reward": 0.37392578125,
|
||
|
|
"rewards/brier_reward": 0.7883727312088012,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4257929682731628,
|
||
|
|
"rewards/smce": -0.0688736081123352,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 589.0,
|
||
|
|
"completions/max_terminated_length": 358.6,
|
||
|
|
"completions/mean_length": 150.88056640625,
|
||
|
|
"completions/mean_terminated_length": 150.74555358886718,
|
||
|
|
"completions/min_length": 70.4,
|
||
|
|
"completions/min_terminated_length": 70.4,
|
||
|
|
"epoch": 0.48,
|
||
|
|
"grad_norm": 0.0009045750484801829,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 499109925.0,
|
||
|
|
"reward": 1.1116684198379516,
|
||
|
|
"reward_std": 0.10544286221265793,
|
||
|
|
"rewards/accuracy_reward": 0.4521484375,
|
||
|
|
"rewards/brier_reward": 0.7713385105133057,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4053613364696503,
|
||
|
|
"rewards/smce": -0.0533256433904171,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.48,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 330.5,
|
||
|
|
"eval_completions/max_terminated_length": 330.5,
|
||
|
|
"eval_completions/mean_length": 151.33397674560547,
|
||
|
|
"eval_completions/mean_terminated_length": 151.33397674560547,
|
||
|
|
"eval_completions/min_length": 86.5,
|
||
|
|
"eval_completions/min_terminated_length": 86.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 499109925.0,
|
||
|
|
"eval_reward": 1.0832970142364502,
|
||
|
|
"eval_reward_std": 0.21396854519844055,
|
||
|
|
"eval_rewards/accuracy_reward": 0.37109375,
|
||
|
|
"eval_rewards/brier_reward": 0.7955322265625,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.40166015923023224,
|
||
|
|
"eval_rewards/smce": -0.03275391226634383,
|
||
|
|
"eval_runtime": 11.706,
|
||
|
|
"eval_samples_per_second": 42.713,
|
||
|
|
"eval_steps_per_second": 0.171,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 633.4,
|
||
|
|
"completions/max_terminated_length": 399.6,
|
||
|
|
"completions/mean_length": 146.53203125,
|
||
|
|
"completions/mean_terminated_length": 146.39669189453124,
|
||
|
|
"completions/min_length": 75.2,
|
||
|
|
"completions/min_terminated_length": 75.2,
|
||
|
|
"epoch": 0.496,
|
||
|
|
"grad_norm": 0.001076922519132495,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 515918253.0,
|
||
|
|
"reward": 1.1104183912277221,
|
||
|
|
"reward_std": 0.11128229051828384,
|
||
|
|
"rewards/accuracy_reward": 0.44697265625,
|
||
|
|
"rewards/brier_reward": 0.7739948391914367,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4203671872615814,
|
||
|
|
"rewards/smce": -0.03376689273864031,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 396.8,
|
||
|
|
"completions/max_terminated_length": 396.8,
|
||
|
|
"completions/mean_length": 146.97705078125,
|
||
|
|
"completions/mean_terminated_length": 146.97705078125,
|
||
|
|
"completions/min_length": 73.0,
|
||
|
|
"completions/min_terminated_length": 73.0,
|
||
|
|
"epoch": 0.512,
|
||
|
|
"grad_norm": 0.000939170247875154,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 532568962.0,
|
||
|
|
"reward": 1.1241384267807006,
|
||
|
|
"reward_std": 0.1158647045493126,
|
||
|
|
"rewards/accuracy_reward": 0.46650390625,
|
||
|
|
"rewards/brier_reward": 0.7818114757537842,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.46039745807647703,
|
||
|
|
"rewards/smce": -0.039459402486681935,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 599.0,
|
||
|
|
"completions/max_terminated_length": 371.8,
|
||
|
|
"completions/mean_length": 153.444921875,
|
||
|
|
"completions/mean_terminated_length": 153.31015014648438,
|
||
|
|
"completions/min_length": 77.6,
|
||
|
|
"completions/min_terminated_length": 77.6,
|
||
|
|
"epoch": 0.528,
|
||
|
|
"grad_norm": 0.0010173226473852992,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 549169774.0,
|
||
|
|
"reward": 1.115717363357544,
|
||
|
|
"reward_std": 0.12228884994983673,
|
||
|
|
"rewards/accuracy_reward": 0.443359375,
|
||
|
|
"rewards/brier_reward": 0.7883172273635864,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.48874316215515134,
|
||
|
|
"rewards/smce": -0.047541006654500964,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 354.0,
|
||
|
|
"completions/max_terminated_length": 354.0,
|
||
|
|
"completions/mean_length": 154.3060546875,
|
||
|
|
"completions/mean_terminated_length": 154.3060546875,
|
||
|
|
"completions/min_length": 72.4,
|
||
|
|
"completions/min_terminated_length": 72.4,
|
||
|
|
"epoch": 0.544,
|
||
|
|
"grad_norm": 0.000949379347730428,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 565913452.0,
|
||
|
|
"reward": 1.1308411598205566,
|
||
|
|
"reward_std": 0.12379086166620254,
|
||
|
|
"rewards/accuracy_reward": 0.48779296875,
|
||
|
|
"rewards/brier_reward": 0.7740159749984741,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4975504994392395,
|
||
|
|
"rewards/smce": -0.029937567189335823,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 449.4,
|
||
|
|
"completions/max_terminated_length": 449.4,
|
||
|
|
"completions/mean_length": 152.90234375,
|
||
|
|
"completions/mean_terminated_length": 152.90234375,
|
||
|
|
"completions/min_length": 75.0,
|
||
|
|
"completions/min_terminated_length": 75.0,
|
||
|
|
"epoch": 0.56,
|
||
|
|
"grad_norm": 0.0007226327434182167,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 582300580.0,
|
||
|
|
"reward": 1.1068236112594605,
|
||
|
|
"reward_std": 0.10674239695072174,
|
||
|
|
"rewards/accuracy_reward": 0.4203125,
|
||
|
|
"rewards/brier_reward": 0.793374526500702,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4588545083999634,
|
||
|
|
"rewards/smce": -0.04062653072178364,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 405.6,
|
||
|
|
"completions/max_terminated_length": 405.6,
|
||
|
|
"completions/mean_length": 154.55283203125,
|
||
|
|
"completions/mean_terminated_length": 154.55283203125,
|
||
|
|
"completions/min_length": 75.0,
|
||
|
|
"completions/min_terminated_length": 75.0,
|
||
|
|
"epoch": 0.576,
|
||
|
|
"grad_norm": 0.0007687191246077418,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0001,
|
||
|
|
"num_tokens": 599069825.0,
|
||
|
|
"reward": 1.1068036079406738,
|
||
|
|
"reward_std": 0.08727760165929795,
|
||
|
|
"rewards/accuracy_reward": 0.4296875,
|
||
|
|
"rewards/brier_reward": 0.7839844107627869,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.3939257860183716,
|
||
|
|
"rewards/smce": -0.06543606072664261,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 681.2,
|
||
|
|
"completions/max_terminated_length": 451.2,
|
||
|
|
"completions/mean_length": 157.06923828125,
|
||
|
|
"completions/mean_terminated_length": 156.93432006835937,
|
||
|
|
"completions/min_length": 77.8,
|
||
|
|
"completions/min_terminated_length": 77.8,
|
||
|
|
"epoch": 0.592,
|
||
|
|
"grad_norm": 0.0007763951434753835,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 615845926.0,
|
||
|
|
"reward": 1.1132864475250244,
|
||
|
|
"reward_std": 0.09145474284887314,
|
||
|
|
"rewards/accuracy_reward": 0.4427734375,
|
||
|
|
"rewards/brier_reward": 0.7840348005294799,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4050097703933716,
|
||
|
|
"rewards/smce": -0.04077463261783123,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 839.2,
|
||
|
|
"completions/max_terminated_length": 394.2,
|
||
|
|
"completions/mean_length": 160.98759765625,
|
||
|
|
"completions/mean_terminated_length": 160.71965942382812,
|
||
|
|
"completions/min_length": 78.8,
|
||
|
|
"completions/min_terminated_length": 78.8,
|
||
|
|
"epoch": 0.608,
|
||
|
|
"grad_norm": 0.0008273112471215427,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 632493927.0,
|
||
|
|
"reward": 1.1193994283676147,
|
||
|
|
"reward_std": 0.10334119200706482,
|
||
|
|
"rewards/accuracy_reward": 0.43974609375,
|
||
|
|
"rewards/brier_reward": 0.7992767453193664,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.43347753286361695,
|
||
|
|
"rewards/smce": -0.02950924411416054,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 662.8,
|
||
|
|
"completions/max_terminated_length": 487.4,
|
||
|
|
"completions/mean_length": 166.30029296875,
|
||
|
|
"completions/mean_terminated_length": 166.16661071777344,
|
||
|
|
"completions/min_length": 81.6,
|
||
|
|
"completions/min_terminated_length": 81.6,
|
||
|
|
"epoch": 0.624,
|
||
|
|
"grad_norm": 0.001086984295397997,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 649540746.0,
|
||
|
|
"reward": 1.110041618347168,
|
||
|
|
"reward_std": 0.11234649866819382,
|
||
|
|
"rewards/accuracy_reward": 0.43046875,
|
||
|
|
"rewards/brier_reward": 0.7898535251617431,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4636425793170929,
|
||
|
|
"rewards/smce": -0.04460376352071762,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 415.8,
|
||
|
|
"completions/max_terminated_length": 415.8,
|
||
|
|
"completions/mean_length": 166.65166015625,
|
||
|
|
"completions/mean_terminated_length": 166.65166015625,
|
||
|
|
"completions/min_length": 85.2,
|
||
|
|
"completions/min_terminated_length": 85.2,
|
||
|
|
"epoch": 0.64,
|
||
|
|
"grad_norm": 0.0008566314354538918,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 666589947.0,
|
||
|
|
"reward": 1.1489856958389282,
|
||
|
|
"reward_std": 0.10852711945772171,
|
||
|
|
"rewards/accuracy_reward": 0.50615234375,
|
||
|
|
"rewards/brier_reward": 0.7918631672859192,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.49933398962020875,
|
||
|
|
"rewards/smce": -0.045029871026054026,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.64,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 366.0,
|
||
|
|
"eval_completions/max_terminated_length": 366.0,
|
||
|
|
"eval_completions/mean_length": 166.41905212402344,
|
||
|
|
"eval_completions/mean_terminated_length": 166.41905212402344,
|
||
|
|
"eval_completions/min_length": 86.0,
|
||
|
|
"eval_completions/min_terminated_length": 86.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 666589947.0,
|
||
|
|
"eval_reward": 1.0801602602005005,
|
||
|
|
"eval_reward_std": 0.2655382305383682,
|
||
|
|
"eval_rewards/accuracy_reward": 0.3671875,
|
||
|
|
"eval_rewards/brier_reward": 0.7932590246200562,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4937109351158142,
|
||
|
|
"eval_rewards/smce": -0.12693358957767487,
|
||
|
|
"eval_runtime": 12.0819,
|
||
|
|
"eval_samples_per_second": 41.384,
|
||
|
|
"eval_steps_per_second": 0.166,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 873.0,
|
||
|
|
"completions/max_terminated_length": 431.8,
|
||
|
|
"completions/mean_length": 163.4501953125,
|
||
|
|
"completions/mean_terminated_length": 163.04900207519532,
|
||
|
|
"completions/min_length": 84.6,
|
||
|
|
"completions/min_terminated_length": 84.6,
|
||
|
|
"epoch": 0.656,
|
||
|
|
"grad_norm": 0.0011378537165001035,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 683120221.0,
|
||
|
|
"reward": 1.1000010251998902,
|
||
|
|
"reward_std": 0.11923554241657257,
|
||
|
|
"rewards/accuracy_reward": 0.4224609375,
|
||
|
|
"rewards/brier_reward": 0.7779239058494568,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.5125224709510803,
|
||
|
|
"rewards/smce": -0.09081879109144211,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 1085.8,
|
||
|
|
"completions/max_terminated_length": 413.4,
|
||
|
|
"completions/mean_length": 163.33291015625,
|
||
|
|
"completions/mean_terminated_length": 162.79642333984376,
|
||
|
|
"completions/min_length": 78.6,
|
||
|
|
"completions/min_terminated_length": 78.6,
|
||
|
|
"epoch": 0.672,
|
||
|
|
"grad_norm": 0.0006824737647548318,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 699706190.0,
|
||
|
|
"reward": 1.1262107133865356,
|
||
|
|
"reward_std": 0.10164524465799332,
|
||
|
|
"rewards/accuracy_reward": 0.4546875,
|
||
|
|
"rewards/brier_reward": 0.7982605934143067,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4685644626617432,
|
||
|
|
"rewards/smce": -0.03929471783339977,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 375.2,
|
||
|
|
"completions/max_terminated_length": 375.2,
|
||
|
|
"completions/mean_length": 161.79052734375,
|
||
|
|
"completions/mean_terminated_length": 161.79052734375,
|
||
|
|
"completions/min_length": 75.4,
|
||
|
|
"completions/min_terminated_length": 75.4,
|
||
|
|
"epoch": 0.688,
|
||
|
|
"grad_norm": 0.0009157939930446446,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0001,
|
||
|
|
"num_tokens": 716316845.0,
|
||
|
|
"reward": 1.1289687395095824,
|
||
|
|
"reward_std": 0.09724249839782714,
|
||
|
|
"rewards/accuracy_reward": 0.4564453125,
|
||
|
|
"rewards/brier_reward": 0.8015339732170105,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4269199311733246,
|
||
|
|
"rewards/smce": -0.04264727309346199,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 865.2,
|
||
|
|
"completions/max_terminated_length": 491.2,
|
||
|
|
"completions/mean_length": 162.37724609375,
|
||
|
|
"completions/mean_terminated_length": 162.10917053222656,
|
||
|
|
"completions/min_length": 84.2,
|
||
|
|
"completions/min_terminated_length": 84.2,
|
||
|
|
"epoch": 0.704,
|
||
|
|
"grad_norm": 0.0008008752483874559,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 732845732.0,
|
||
|
|
"reward": 1.1373294115066528,
|
||
|
|
"reward_std": 0.0963486909866333,
|
||
|
|
"rewards/accuracy_reward": 0.4806640625,
|
||
|
|
"rewards/brier_reward": 0.7942330479621887,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.43858984112739563,
|
||
|
|
"rewards/smce": -0.04382671192288399,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 857.6,
|
||
|
|
"completions/max_terminated_length": 392.6,
|
||
|
|
"completions/mean_length": 161.50390625,
|
||
|
|
"completions/mean_terminated_length": 161.1012969970703,
|
||
|
|
"completions/min_length": 69.6,
|
||
|
|
"completions/min_terminated_length": 69.6,
|
||
|
|
"epoch": 0.72,
|
||
|
|
"grad_norm": 0.000978630967438221,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 749509388.0,
|
||
|
|
"reward": 1.1495690107345582,
|
||
|
|
"reward_std": 0.10505697280168533,
|
||
|
|
"rewards/accuracy_reward": 0.4892578125,
|
||
|
|
"rewards/brier_reward": 0.8102994561195374,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.49882130026817323,
|
||
|
|
"rewards/smce": -0.02960990173742175,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 826.8,
|
||
|
|
"completions/max_terminated_length": 391.4,
|
||
|
|
"completions/mean_length": 159.55869140625,
|
||
|
|
"completions/mean_terminated_length": 159.15543518066406,
|
||
|
|
"completions/min_length": 83.0,
|
||
|
|
"completions/min_terminated_length": 83.0,
|
||
|
|
"epoch": 0.736,
|
||
|
|
"grad_norm": 0.0009836617391556501,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 766082853.0,
|
||
|
|
"reward": 1.1441120862960816,
|
||
|
|
"reward_std": 0.10725852996110916,
|
||
|
|
"rewards/accuracy_reward": 0.496484375,
|
||
|
|
"rewards/brier_reward": 0.7920857787132263,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.5160800814628601,
|
||
|
|
"rewards/smce": -0.05401993244886398,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 600.8,
|
||
|
|
"completions/max_terminated_length": 384.6,
|
||
|
|
"completions/mean_length": 159.32666015625,
|
||
|
|
"completions/mean_terminated_length": 159.19202270507813,
|
||
|
|
"completions/min_length": 76.2,
|
||
|
|
"completions/min_terminated_length": 76.2,
|
||
|
|
"epoch": 0.752,
|
||
|
|
"grad_norm": 0.0010423972271382809,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 782941558.0,
|
||
|
|
"reward": 1.1330698251724243,
|
||
|
|
"reward_std": 0.10301287025213242,
|
||
|
|
"rewards/accuracy_reward": 0.470703125,
|
||
|
|
"rewards/brier_reward": 0.7955723881721497,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4687236487865448,
|
||
|
|
"rewards/smce": -0.039099032431840895,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 858.4,
|
||
|
|
"completions/max_terminated_length": 438.0,
|
||
|
|
"completions/mean_length": 165.8583984375,
|
||
|
|
"completions/mean_terminated_length": 165.32333984375,
|
||
|
|
"completions/min_length": 80.0,
|
||
|
|
"completions/min_terminated_length": 80.0,
|
||
|
|
"epoch": 0.768,
|
||
|
|
"grad_norm": 0.0008880188688635826,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 799572652.0,
|
||
|
|
"reward": 1.1417131900787354,
|
||
|
|
"reward_std": 0.0946048617362976,
|
||
|
|
"rewards/accuracy_reward": 0.4765625,
|
||
|
|
"rewards/brier_reward": 0.8072758674621582,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4606347680091858,
|
||
|
|
"rewards/smce": -0.02230907715857029,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 455.6,
|
||
|
|
"completions/max_terminated_length": 455.6,
|
||
|
|
"completions/mean_length": 166.615234375,
|
||
|
|
"completions/mean_terminated_length": 166.615234375,
|
||
|
|
"completions/min_length": 81.0,
|
||
|
|
"completions/min_terminated_length": 81.0,
|
||
|
|
"epoch": 0.784,
|
||
|
|
"grad_norm": 0.0008837269269861281,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 816453160.0,
|
||
|
|
"reward": 1.142217469215393,
|
||
|
|
"reward_std": 0.10188348740339279,
|
||
|
|
"rewards/accuracy_reward": 0.49140625,
|
||
|
|
"rewards/brier_reward": 0.7931073784828186,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4713378846645355,
|
||
|
|
"rewards/smce": -0.07962073720991611,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 1034.2,
|
||
|
|
"completions/max_terminated_length": 594.8,
|
||
|
|
"completions/mean_length": 165.168359375,
|
||
|
|
"completions/mean_terminated_length": 164.90091857910156,
|
||
|
|
"completions/min_length": 85.0,
|
||
|
|
"completions/min_terminated_length": 85.0,
|
||
|
|
"epoch": 0.8,
|
||
|
|
"grad_norm": 0.001023618970066309,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 833155044.0,
|
||
|
|
"reward": 1.1532862901687622,
|
||
|
|
"reward_std": 0.11082861423492432,
|
||
|
|
"rewards/accuracy_reward": 0.50244140625,
|
||
|
|
"rewards/brier_reward": 0.8043578028678894,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5133701205253601,
|
||
|
|
"rewards/smce": -0.03228470087051392,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 346.0,
|
||
|
|
"eval_completions/max_terminated_length": 346.0,
|
||
|
|
"eval_completions/mean_length": 169.8637924194336,
|
||
|
|
"eval_completions/mean_terminated_length": 169.8637924194336,
|
||
|
|
"eval_completions/min_length": 96.0,
|
||
|
|
"eval_completions/min_terminated_length": 96.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 833155044.0,
|
||
|
|
"eval_reward": 1.0787333250045776,
|
||
|
|
"eval_reward_std": 0.2926654517650604,
|
||
|
|
"eval_rewards/accuracy_reward": 0.376953125,
|
||
|
|
"eval_rewards/brier_reward": 0.7806787192821503,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.5430664122104645,
|
||
|
|
"eval_rewards/smce": -0.16611327975988388,
|
||
|
|
"eval_runtime": 12.1823,
|
||
|
|
"eval_samples_per_second": 41.043,
|
||
|
|
"eval_steps_per_second": 0.164,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 589.6,
|
||
|
|
"completions/max_terminated_length": 589.6,
|
||
|
|
"completions/mean_length": 160.68017578125,
|
||
|
|
"completions/mean_terminated_length": 160.68017578125,
|
||
|
|
"completions/min_length": 82.2,
|
||
|
|
"completions/min_terminated_length": 82.2,
|
||
|
|
"epoch": 0.816,
|
||
|
|
"grad_norm": 0.0013014374999329448,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0002,
|
||
|
|
"num_tokens": 849899577.0,
|
||
|
|
"reward": 1.144900870323181,
|
||
|
|
"reward_std": 0.12142248898744583,
|
||
|
|
"rewards/accuracy_reward": 0.49375,
|
||
|
|
"rewards/brier_reward": 0.796118152141571,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.5607226490974426,
|
||
|
|
"rewards/smce": -0.06757495403289795,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 644.2,
|
||
|
|
"completions/max_terminated_length": 443.2,
|
||
|
|
"completions/mean_length": 163.26494140625,
|
||
|
|
"completions/mean_terminated_length": 163.13106994628907,
|
||
|
|
"completions/min_length": 83.2,
|
||
|
|
"completions/min_terminated_length": 83.2,
|
||
|
|
"epoch": 0.832,
|
||
|
|
"grad_norm": 0.0008710598922334611,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 866579762.0,
|
||
|
|
"reward": 1.1563215732574463,
|
||
|
|
"reward_std": 0.09912585914134979,
|
||
|
|
"rewards/accuracy_reward": 0.50263671875,
|
||
|
|
"rewards/brier_reward": 0.8101438045501709,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4804052829742432,
|
||
|
|
"rewards/smce": -0.04059399105608463,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 390.2,
|
||
|
|
"completions/max_terminated_length": 390.2,
|
||
|
|
"completions/mean_length": 160.62734375,
|
||
|
|
"completions/mean_terminated_length": 160.62734375,
|
||
|
|
"completions/min_length": 83.2,
|
||
|
|
"completions/min_terminated_length": 83.2,
|
||
|
|
"epoch": 0.848,
|
||
|
|
"grad_norm": 0.0011734205763787031,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 883238954.0,
|
||
|
|
"reward": 1.139661955833435,
|
||
|
|
"reward_std": 0.0918037623167038,
|
||
|
|
"rewards/accuracy_reward": 0.47666015625,
|
||
|
|
"rewards/brier_reward": 0.8027248620986939,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4152099609375,
|
||
|
|
"rewards/smce": -0.06195409372448921,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 678.8,
|
||
|
|
"completions/max_terminated_length": 450.0,
|
||
|
|
"completions/mean_length": 168.620703125,
|
||
|
|
"completions/mean_terminated_length": 168.48779296875,
|
||
|
|
"completions/min_length": 85.2,
|
||
|
|
"completions/min_terminated_length": 85.2,
|
||
|
|
"epoch": 0.864,
|
||
|
|
"grad_norm": 0.0009395951055921614,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 899952446.0,
|
||
|
|
"reward": 1.150853943824768,
|
||
|
|
"reward_std": 0.09571457803249359,
|
||
|
|
"rewards/accuracy_reward": 0.5037109375,
|
||
|
|
"rewards/brier_reward": 0.7982314586639404,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4721386730670929,
|
||
|
|
"rewards/smce": -0.04000121615827083,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 460.8,
|
||
|
|
"completions/max_terminated_length": 460.8,
|
||
|
|
"completions/mean_length": 176.70556640625,
|
||
|
|
"completions/mean_terminated_length": 176.70556640625,
|
||
|
|
"completions/min_length": 81.8,
|
||
|
|
"completions/min_terminated_length": 81.8,
|
||
|
|
"epoch": 0.88,
|
||
|
|
"grad_norm": 0.001014765352010727,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0,
|
||
|
|
"num_tokens": 916908983.0,
|
||
|
|
"reward": 1.1355210781097411,
|
||
|
|
"reward_std": 0.09766919016838074,
|
||
|
|
"rewards/accuracy_reward": 0.46201171875,
|
||
|
|
"rewards/brier_reward": 0.8090754985809326,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4960673928260803,
|
||
|
|
"rewards/smce": -0.04611279219388962,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 871.6,
|
||
|
|
"completions/max_terminated_length": 611.4,
|
||
|
|
"completions/mean_length": 184.18115234375,
|
||
|
|
"completions/mean_terminated_length": 183.91699829101563,
|
||
|
|
"completions/min_length": 91.6,
|
||
|
|
"completions/min_terminated_length": 91.6,
|
||
|
|
"epoch": 0.896,
|
||
|
|
"grad_norm": 0.0010061163920909166,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 933905846.0,
|
||
|
|
"reward": 1.1420814514160156,
|
||
|
|
"reward_std": 0.09720710813999175,
|
||
|
|
"rewards/accuracy_reward": 0.478125,
|
||
|
|
"rewards/brier_reward": 0.8063697218894958,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.48104395270347594,
|
||
|
|
"rewards/smce": -0.039731436781585215,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1086.6,
|
||
|
|
"completions/max_terminated_length": 438.4,
|
||
|
|
"completions/mean_length": 186.08662109375,
|
||
|
|
"completions/mean_terminated_length": 185.6914276123047,
|
||
|
|
"completions/min_length": 94.8,
|
||
|
|
"completions/min_terminated_length": 94.8,
|
||
|
|
"epoch": 0.912,
|
||
|
|
"grad_norm": 0.00092413614038378,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 950862669.0,
|
||
|
|
"reward": 1.13735990524292,
|
||
|
|
"reward_std": 0.09484986811876298,
|
||
|
|
"rewards/accuracy_reward": 0.47255859375,
|
||
|
|
"rewards/brier_reward": 0.8024985313415527,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4633896589279175,
|
||
|
|
"rewards/smce": -0.04536058492958546,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 1162.8,
|
||
|
|
"completions/max_terminated_length": 554.2,
|
||
|
|
"completions/mean_length": 186.5712890625,
|
||
|
|
"completions/mean_terminated_length": 186.04344482421874,
|
||
|
|
"completions/min_length": 91.0,
|
||
|
|
"completions/min_terminated_length": 91.0,
|
||
|
|
"epoch": 0.928,
|
||
|
|
"grad_norm": 0.0008485357975587249,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"num_tokens": 967799975.0,
|
||
|
|
"reward": 1.1491246938705444,
|
||
|
|
"reward_std": 0.08749246448278428,
|
||
|
|
"rewards/accuracy_reward": 0.505859375,
|
||
|
|
"rewards/brier_reward": 0.7928359508514404,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.45264649391174316,
|
||
|
|
"rewards/smce": -0.05614354871213436,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1129.4,
|
||
|
|
"completions/max_terminated_length": 641.6,
|
||
|
|
"completions/mean_length": 187.45673828125,
|
||
|
|
"completions/mean_terminated_length": 187.06110534667968,
|
||
|
|
"completions/min_length": 89.2,
|
||
|
|
"completions/min_terminated_length": 89.2,
|
||
|
|
"epoch": 0.944,
|
||
|
|
"grad_norm": 0.0009085385827347636,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"num_tokens": 984694956.0,
|
||
|
|
"reward": 1.1451599836349486,
|
||
|
|
"reward_std": 0.09136117100715638,
|
||
|
|
"rewards/accuracy_reward": 0.4962890625,
|
||
|
|
"rewards/brier_reward": 0.7944116115570068,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4501757860183716,
|
||
|
|
"rewards/smce": -0.0885808089748025,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 755.6,
|
||
|
|
"completions/max_terminated_length": 587.0,
|
||
|
|
"completions/mean_length": 187.70068359375,
|
||
|
|
"completions/mean_terminated_length": 187.43772583007814,
|
||
|
|
"completions/min_length": 91.4,
|
||
|
|
"completions/min_terminated_length": 91.4,
|
||
|
|
"epoch": 0.96,
|
||
|
|
"grad_norm": 0.000844655092805624,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 1001557331.0,
|
||
|
|
"reward": 1.1376837253570558,
|
||
|
|
"reward_std": 0.0915878638625145,
|
||
|
|
"rewards/accuracy_reward": 0.4642578125,
|
||
|
|
"rewards/brier_reward": 0.8113359332084655,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4858310461044312,
|
||
|
|
"rewards/smce": -0.031868165358901024,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.96,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 392.5,
|
||
|
|
"eval_completions/max_terminated_length": 392.5,
|
||
|
|
"eval_completions/mean_length": 191.19613647460938,
|
||
|
|
"eval_completions/mean_terminated_length": 191.19613647460938,
|
||
|
|
"eval_completions/min_length": 92.0,
|
||
|
|
"eval_completions/min_terminated_length": 92.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 1001557331.0,
|
||
|
|
"eval_reward": 1.1049875020980835,
|
||
|
|
"eval_reward_std": 0.26456505060195923,
|
||
|
|
"eval_rewards/accuracy_reward": 0.40625,
|
||
|
|
"eval_rewards/brier_reward": 0.803798645734787,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4793164134025574,
|
||
|
|
"eval_rewards/smce": -0.07452617585659027,
|
||
|
|
"eval_runtime": 12.6872,
|
||
|
|
"eval_samples_per_second": 39.41,
|
||
|
|
"eval_steps_per_second": 0.158,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 958.4,
|
||
|
|
"completions/max_terminated_length": 630.4,
|
||
|
|
"completions/mean_length": 189.08857421875,
|
||
|
|
"completions/mean_terminated_length": 188.6948211669922,
|
||
|
|
"completions/min_length": 96.4,
|
||
|
|
"completions/min_terminated_length": 96.4,
|
||
|
|
"epoch": 0.976,
|
||
|
|
"grad_norm": 0.000775482039898634,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 1018354718.0,
|
||
|
|
"reward": 1.1589978218078614,
|
||
|
|
"reward_std": 0.08740981668233871,
|
||
|
|
"rewards/accuracy_reward": 0.50595703125,
|
||
|
|
"rewards/brier_reward": 0.8123676776885986,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4781347870826721,
|
||
|
|
"rewards/smce": -0.037019770964980124,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0005859375,
|
||
|
|
"completions/max_length": 1359.8,
|
||
|
|
"completions/max_terminated_length": 520.6,
|
||
|
|
"completions/mean_length": 187.88623046875,
|
||
|
|
"completions/mean_terminated_length": 187.09657287597656,
|
||
|
|
"completions/min_length": 92.6,
|
||
|
|
"completions/min_terminated_length": 92.6,
|
||
|
|
"epoch": 0.992,
|
||
|
|
"grad_norm": 0.0007895891321823001,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"num_tokens": 1035407153.0,
|
||
|
|
"reward": 1.1390615701675415,
|
||
|
|
"reward_std": 0.09859338849782943,
|
||
|
|
"rewards/accuracy_reward": 0.4849609375,
|
||
|
|
"rewards/brier_reward": 0.7937985897064209,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.9994140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.44668457508087156,
|
||
|
|
"rewards/smce": -0.05136181376874447,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000732421875,
|
||
|
|
"completions/max_length": 989.0,
|
||
|
|
"completions/max_terminated_length": 730.5,
|
||
|
|
"completions/mean_length": 187.27367401123047,
|
||
|
|
"completions/mean_terminated_length": 186.28497314453125,
|
||
|
|
"completions/min_length": 88.0,
|
||
|
|
"completions/min_terminated_length": 88.0,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"num_tokens": 1042184222.0,
|
||
|
|
"reward": 1.138842523097992,
|
||
|
|
"reward_std": 0.09942503273487091,
|
||
|
|
"rewards/accuracy_reward": 0.478759765625,
|
||
|
|
"rewards/brier_reward": 0.7997052073478699,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999267578125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4984985440969467,
|
||
|
|
"rewards/smce": -0.048381347209215164,
|
||
|
|
"step": 312,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.004145986298401392,
|
||
|
|
"train_runtime": 42424.6813,
|
||
|
|
"train_samples_per_second": 0.471,
|
||
|
|
"train_steps_per_second": 0.007
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 312,
|
||
|
|
"num_input_tokens_seen": 1042184222,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 60,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 4,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|