1944 lines
72 KiB
JSON
1944 lines
72 KiB
JSON
|
{
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"eval_steps": 50,
|
||
|
|
"global_step": 312,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.11044921875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1529.0,
|
||
|
|
"completions/mean_length": 472.82919921875,
|
||
|
|
"completions/mean_terminated_length": 340.9787963867187,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.016,
|
||
|
|
"grad_norm": 0.004426147323101759,
|
||
|
|
"learning_rate": 3.1249999999999997e-07,
|
||
|
|
"loss": 0.0444,
|
||
|
|
"num_tokens": 19810059.0,
|
||
|
|
"reward": 0.20038452744483948,
|
||
|
|
"reward_std": 0.36961849331855773,
|
||
|
|
"rewards/accuracy_reward": 0.053515625,
|
||
|
|
"rewards/brier_reward": 0.09974026381969452,
|
||
|
|
"rewards/confidence_one_or_zero": 0.401953125,
|
||
|
|
"rewards/format_reward": 0.24755859375,
|
||
|
|
"rewards/mean_confidence_reward": 0.5259665191173554,
|
||
|
|
"rewards/smce": -0.47275206446647644,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.09716796875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1524.2,
|
||
|
|
"completions/mean_length": 431.33447265625,
|
||
|
|
"completions/mean_terminated_length": 312.50458984375,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.032,
|
||
|
|
"grad_norm": 0.003934022039175034,
|
||
|
|
"learning_rate": 6.249999999999999e-07,
|
||
|
|
"loss": 0.0498,
|
||
|
|
"num_tokens": 39265452.0,
|
||
|
|
"reward": 0.2235296219587326,
|
||
|
|
"reward_std": 0.3799919366836548,
|
||
|
|
"rewards/accuracy_reward": 0.0552734375,
|
||
|
|
"rewards/brier_reward": 0.10921734869480133,
|
||
|
|
"rewards/confidence_one_or_zero": 0.424609375,
|
||
|
|
"rewards/format_reward": 0.2826171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5619598150253295,
|
||
|
|
"rewards/smce": -0.5068929612636566,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.054296875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1487.0,
|
||
|
|
"completions/mean_length": 312.63193359375,
|
||
|
|
"completions/mean_terminated_length": 242.80802307128906,
|
||
|
|
"completions/min_length": 1.8,
|
||
|
|
"completions/min_terminated_length": 1.8,
|
||
|
|
"epoch": 0.048,
|
||
|
|
"grad_norm": 0.00401897681877017,
|
||
|
|
"learning_rate": 9.374999999999999e-07,
|
||
|
|
"loss": 0.0483,
|
||
|
|
"num_tokens": 57529843.0,
|
||
|
|
"reward": 0.4077006340026855,
|
||
|
|
"reward_std": 0.4407335460186005,
|
||
|
|
"rewards/accuracy_reward": 0.10361328125,
|
||
|
|
"rewards/brier_reward": 0.19856337010860442,
|
||
|
|
"rewards/confidence_one_or_zero": 0.53974609375,
|
||
|
|
"rewards/format_reward": 0.51328125,
|
||
|
|
"rewards/mean_confidence_reward": 0.6942256689071655,
|
||
|
|
"rewards/smce": -0.5910538554191589,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.021484375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1397.2,
|
||
|
|
"completions/mean_length": 214.742578125,
|
||
|
|
"completions/mean_terminated_length": 185.78294677734374,
|
||
|
|
"completions/min_length": 16.2,
|
||
|
|
"completions/min_terminated_length": 16.2,
|
||
|
|
"epoch": 0.064,
|
||
|
|
"grad_norm": 0.002913103671744466,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.037,
|
||
|
|
"num_tokens": 74777895.0,
|
||
|
|
"reward": 0.6778669238090516,
|
||
|
|
"reward_std": 0.39471102356910703,
|
||
|
|
"rewards/accuracy_reward": 0.187109375,
|
||
|
|
"rewards/brier_reward": 0.3357722103595734,
|
||
|
|
"rewards/confidence_one_or_zero": 0.5685546875,
|
||
|
|
"rewards/format_reward": 0.83291015625,
|
||
|
|
"rewards/mean_confidence_reward": 0.7907521605491639,
|
||
|
|
"rewards/smce": -0.6059021592140198,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.01279296875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1216.0,
|
||
|
|
"completions/mean_length": 165.07568359375,
|
||
|
|
"completions/mean_terminated_length": 147.32185974121094,
|
||
|
|
"completions/min_length": 21.6,
|
||
|
|
"completions/min_terminated_length": 21.6,
|
||
|
|
"epoch": 0.08,
|
||
|
|
"grad_norm": 0.002816141350194812,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0187,
|
||
|
|
"num_tokens": 91500174.0,
|
||
|
|
"reward": 0.7974048018455505,
|
||
|
|
"reward_std": 0.3286789894104004,
|
||
|
|
"rewards/accuracy_reward": 0.21787109375,
|
||
|
|
"rewards/brier_reward": 0.4181051731109619,
|
||
|
|
"rewards/confidence_one_or_zero": 0.476953125,
|
||
|
|
"rewards/format_reward": 0.95888671875,
|
||
|
|
"rewards/mean_confidence_reward": 0.7712665319442749,
|
||
|
|
"rewards/smce": -0.5586693763732911,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00751953125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1278.6,
|
||
|
|
"completions/mean_length": 142.0654296875,
|
||
|
|
"completions/mean_terminated_length": 131.50679779052734,
|
||
|
|
"completions/min_length": 13.6,
|
||
|
|
"completions/min_terminated_length": 13.6,
|
||
|
|
"epoch": 0.096,
|
||
|
|
"grad_norm": 0.0024108965881168842,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0124,
|
||
|
|
"num_tokens": 107868204.0,
|
||
|
|
"reward": 0.895538580417633,
|
||
|
|
"reward_std": 0.2955020546913147,
|
||
|
|
"rewards/accuracy_reward": 0.25625,
|
||
|
|
"rewards/brier_reward": 0.5535168647766113,
|
||
|
|
"rewards/confidence_one_or_zero": 0.33291015625,
|
||
|
|
"rewards/format_reward": 0.98134765625,
|
||
|
|
"rewards/mean_confidence_reward": 0.6346356272697449,
|
||
|
|
"rewards/smce": -0.39225292205810547,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00380859375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 908.2,
|
||
|
|
"completions/mean_length": 126.67451171875,
|
||
|
|
"completions/mean_terminated_length": 121.2887466430664,
|
||
|
|
"completions/min_length": 37.2,
|
||
|
|
"completions/min_terminated_length": 37.2,
|
||
|
|
"epoch": 0.112,
|
||
|
|
"grad_norm": 0.0012922441819682717,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0069,
|
||
|
|
"num_tokens": 124179495.0,
|
||
|
|
"reward": 0.9732187509536743,
|
||
|
|
"reward_std": 0.2013139009475708,
|
||
|
|
"rewards/accuracy_reward": 0.24404296875,
|
||
|
|
"rewards/brier_reward": 0.7092463850975037,
|
||
|
|
"rewards/confidence_one_or_zero": 0.228125,
|
||
|
|
"rewards/format_reward": 0.9931640625,
|
||
|
|
"rewards/mean_confidence_reward": 0.3768877565860748,
|
||
|
|
"rewards/smce": -0.17112396359443666,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00126953125,
|
||
|
|
"completions/max_length": 1395.8,
|
||
|
|
"completions/max_terminated_length": 780.2,
|
||
|
|
"completions/mean_length": 119.758984375,
|
||
|
|
"completions/mean_terminated_length": 117.95802612304688,
|
||
|
|
"completions/min_length": 47.2,
|
||
|
|
"completions/min_terminated_length": 47.2,
|
||
|
|
"epoch": 0.128,
|
||
|
|
"grad_norm": 0.0008959283004514873,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0023,
|
||
|
|
"num_tokens": 140359299.0,
|
||
|
|
"reward": 1.0186350703239442,
|
||
|
|
"reward_std": 0.13564346730709076,
|
||
|
|
"rewards/accuracy_reward": 0.2814453125,
|
||
|
|
"rewards/brier_reward": 0.7590552687644958,
|
||
|
|
"rewards/confidence_one_or_zero": 0.203125,
|
||
|
|
"rewards/format_reward": 0.99677734375,
|
||
|
|
"rewards/mean_confidence_reward": 0.246020370721817,
|
||
|
|
"rewards/smce": -0.08598535656929016,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00078125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 716.4,
|
||
|
|
"completions/mean_length": 119.24130859375,
|
||
|
|
"completions/mean_terminated_length": 118.13349609375,
|
||
|
|
"completions/min_length": 48.8,
|
||
|
|
"completions/min_terminated_length": 48.8,
|
||
|
|
"epoch": 0.144,
|
||
|
|
"grad_norm": 0.0008299688925035298,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0017,
|
||
|
|
"num_tokens": 156308266.0,
|
||
|
|
"reward": 1.033864641189575,
|
||
|
|
"reward_std": 0.10059063583612442,
|
||
|
|
"rewards/accuracy_reward": 0.30849609375,
|
||
|
|
"rewards/brier_reward": 0.7607093691825867,
|
||
|
|
"rewards/confidence_one_or_zero": 0.15751953125,
|
||
|
|
"rewards/format_reward": 0.99853515625,
|
||
|
|
"rewards/mean_confidence_reward": 0.19957530796527861,
|
||
|
|
"rewards/smce": -0.12014870196580887,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0005859375,
|
||
|
|
"completions/max_length": 1055.6,
|
||
|
|
"completions/max_terminated_length": 644.2,
|
||
|
|
"completions/mean_length": 119.0482421875,
|
||
|
|
"completions/mean_terminated_length": 118.2185302734375,
|
||
|
|
"completions/min_length": 51.2,
|
||
|
|
"completions/min_terminated_length": 51.2,
|
||
|
|
"epoch": 0.16,
|
||
|
|
"grad_norm": 0.000901242543477565,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 172699192.0,
|
||
|
|
"reward": 1.0454564571380616,
|
||
|
|
"reward_std": 0.09141466915607452,
|
||
|
|
"rewards/accuracy_reward": 0.3259765625,
|
||
|
|
"rewards/brier_reward": 0.7662179708480835,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0939453125,
|
||
|
|
"rewards/format_reward": 0.99873046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.2009649395942688,
|
||
|
|
"rewards/smce": -0.12679803520441055,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16,
|
||
|
|
"eval_completions/clipped_ratio": 0.001953125,
|
||
|
|
"eval_completions/max_length": 997.5,
|
||
|
|
"eval_completions/max_terminated_length": 347.5,
|
||
|
|
"eval_completions/mean_length": 127.43029403686523,
|
||
|
|
"eval_completions/mean_terminated_length": 124.67033386230469,
|
||
|
|
"eval_completions/min_length": 55.0,
|
||
|
|
"eval_completions/min_terminated_length": 55.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 172699192.0,
|
||
|
|
"eval_reward": 1.0254148244857788,
|
||
|
|
"eval_reward_std": 0.1284085400402546,
|
||
|
|
"eval_rewards/accuracy_reward": 0.236328125,
|
||
|
|
"eval_rewards/brier_reward": 0.8164560794830322,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.05859375,
|
||
|
|
"eval_rewards/format_reward": 0.998046875,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.22532617300748825,
|
||
|
|
"eval_rewards/smce": -0.020053687505424023,
|
||
|
|
"eval_runtime": 22.7777,
|
||
|
|
"eval_samples_per_second": 21.951,
|
||
|
|
"eval_steps_per_second": 0.088,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 899.4,
|
||
|
|
"completions/max_terminated_length": 439.8,
|
||
|
|
"completions/mean_length": 119.5984375,
|
||
|
|
"completions/mean_terminated_length": 119.32182464599609,
|
||
|
|
"completions/min_length": 56.4,
|
||
|
|
"completions/min_terminated_length": 56.4,
|
||
|
|
"epoch": 0.176,
|
||
|
|
"grad_norm": 0.0008352047298103571,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 188940424.0,
|
||
|
|
"reward": 1.0325161933898925,
|
||
|
|
"reward_std": 0.08973144143819808,
|
||
|
|
"rewards/accuracy_reward": 0.2689453125,
|
||
|
|
"rewards/brier_reward": 0.7969693899154663,
|
||
|
|
"rewards/confidence_one_or_zero": 0.035546875,
|
||
|
|
"rewards/format_reward": 0.99912109375,
|
||
|
|
"rewards/mean_confidence_reward": 0.24108357727527618,
|
||
|
|
"rewards/smce": -0.039200416207313536,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0005859375,
|
||
|
|
"completions/max_length": 1097.4,
|
||
|
|
"completions/max_terminated_length": 780.8,
|
||
|
|
"completions/mean_length": 121.203125,
|
||
|
|
"completions/mean_terminated_length": 120.3740463256836,
|
||
|
|
"completions/min_length": 56.4,
|
||
|
|
"completions/min_terminated_length": 56.4,
|
||
|
|
"epoch": 0.192,
|
||
|
|
"grad_norm": 0.0008346716640517116,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 205123144.0,
|
||
|
|
"reward": 1.0596784591674804,
|
||
|
|
"reward_std": 0.09918512254953385,
|
||
|
|
"rewards/accuracy_reward": 0.35859375,
|
||
|
|
"rewards/brier_reward": 0.761845862865448,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0201171875,
|
||
|
|
"rewards/format_reward": 0.99892578125,
|
||
|
|
"rewards/mean_confidence_reward": 0.26954297423362733,
|
||
|
|
"rewards/smce": -0.09118027314543724,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 628.4,
|
||
|
|
"completions/max_terminated_length": 398.6,
|
||
|
|
"completions/mean_length": 121.64189453125,
|
||
|
|
"completions/mean_terminated_length": 121.50375213623047,
|
||
|
|
"completions/min_length": 58.0,
|
||
|
|
"completions/min_terminated_length": 58.0,
|
||
|
|
"epoch": 0.208,
|
||
|
|
"grad_norm": 0.000969071697909385,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 221270965.0,
|
||
|
|
"reward": 1.069624376296997,
|
||
|
|
"reward_std": 0.10735679119825363,
|
||
|
|
"rewards/accuracy_reward": 0.37919921875,
|
||
|
|
"rewards/brier_reward": 0.7605438590049743,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0107421875,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.3172558665275574,
|
||
|
|
"rewards/smce": -0.06722123511135578,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 857.4,
|
||
|
|
"completions/max_terminated_length": 385.4,
|
||
|
|
"completions/mean_length": 124.721484375,
|
||
|
|
"completions/mean_terminated_length": 124.44608459472656,
|
||
|
|
"completions/min_length": 57.6,
|
||
|
|
"completions/min_terminated_length": 57.6,
|
||
|
|
"epoch": 0.224,
|
||
|
|
"grad_norm": 0.0011017704382538795,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 237510193.0,
|
||
|
|
"reward": 1.055229949951172,
|
||
|
|
"reward_std": 0.11658172458410263,
|
||
|
|
"rewards/accuracy_reward": 0.33193359375,
|
||
|
|
"rewards/brier_reward": 0.779213547706604,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00517578125,
|
||
|
|
"rewards/format_reward": 0.99931640625,
|
||
|
|
"rewards/mean_confidence_reward": 0.356457781791687,
|
||
|
|
"rewards/smce": -0.0435761496424675,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 366.0,
|
||
|
|
"completions/max_terminated_length": 366.0,
|
||
|
|
"completions/mean_length": 128.07626953125,
|
||
|
|
"completions/mean_terminated_length": 128.07626953125,
|
||
|
|
"completions/min_length": 58.8,
|
||
|
|
"completions/min_terminated_length": 58.8,
|
||
|
|
"epoch": 0.24,
|
||
|
|
"grad_norm": 0.0010217519011348486,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 254027358.0,
|
||
|
|
"reward": 1.0620179176330566,
|
||
|
|
"reward_std": 0.11173272877931595,
|
||
|
|
"rewards/accuracy_reward": 0.35087890625,
|
||
|
|
"rewards/brier_reward": 0.7738432168960572,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0037109375,
|
||
|
|
"rewards/format_reward": 0.99931640625,
|
||
|
|
"rewards/mean_confidence_reward": 0.3753812491893768,
|
||
|
|
"rewards/smce": -0.03457403443753719,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 956.8,
|
||
|
|
"completions/max_terminated_length": 460.6,
|
||
|
|
"completions/mean_length": 131.16923828125,
|
||
|
|
"completions/mean_terminated_length": 130.89476623535157,
|
||
|
|
"completions/min_length": 57.8,
|
||
|
|
"completions/min_terminated_length": 57.8,
|
||
|
|
"epoch": 0.256,
|
||
|
|
"grad_norm": 0.0009919044096022844,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 270449027.0,
|
||
|
|
"reward": 1.0897830486297608,
|
||
|
|
"reward_std": 0.11419396549463272,
|
||
|
|
"rewards/accuracy_reward": 0.41181640625,
|
||
|
|
"rewards/brier_reward": 0.7680457592010498,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00185546875,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.38369869589805605,
|
||
|
|
"rewards/smce": -0.03861977197229862,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 685.8,
|
||
|
|
"completions/max_terminated_length": 455.6,
|
||
|
|
"completions/mean_length": 131.84853515625,
|
||
|
|
"completions/mean_terminated_length": 131.7114471435547,
|
||
|
|
"completions/min_length": 56.0,
|
||
|
|
"completions/min_terminated_length": 56.0,
|
||
|
|
"epoch": 0.272,
|
||
|
|
"grad_norm": 0.000989066786132753,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 286827284.0,
|
||
|
|
"reward": 1.0802953958511352,
|
||
|
|
"reward_std": 0.1197711020708084,
|
||
|
|
"rewards/accuracy_reward": 0.394140625,
|
||
|
|
"rewards/brier_reward": 0.7666484951972962,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00126953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.3936794936656952,
|
||
|
|
"rewards/smce": -0.03813821934163571,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 392.6,
|
||
|
|
"completions/max_terminated_length": 392.6,
|
||
|
|
"completions/mean_length": 133.7263671875,
|
||
|
|
"completions/mean_terminated_length": 133.7263671875,
|
||
|
|
"completions/min_length": 59.8,
|
||
|
|
"completions/min_terminated_length": 59.8,
|
||
|
|
"epoch": 0.288,
|
||
|
|
"grad_norm": 0.0009975044522434473,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0,
|
||
|
|
"num_tokens": 303172418.0,
|
||
|
|
"reward": 1.0799544334411622,
|
||
|
|
"reward_std": 0.11399292945861816,
|
||
|
|
"rewards/accuracy_reward": 0.38916015625,
|
||
|
|
"rewards/brier_reward": 0.77084881067276,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0013671875,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.39172295331954954,
|
||
|
|
"rewards/smce": -0.0327066283673048,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 597.6,
|
||
|
|
"completions/max_terminated_length": 364.4,
|
||
|
|
"completions/mean_length": 139.4765625,
|
||
|
|
"completions/mean_terminated_length": 139.34026489257812,
|
||
|
|
"completions/min_length": 64.4,
|
||
|
|
"completions/min_terminated_length": 64.4,
|
||
|
|
"epoch": 0.304,
|
||
|
|
"grad_norm": 0.000925487547647208,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 319776530.0,
|
||
|
|
"reward": 1.0868202686309814,
|
||
|
|
"reward_std": 0.11602218449115753,
|
||
|
|
"rewards/accuracy_reward": 0.41484375,
|
||
|
|
"rewards/brier_reward": 0.759091567993164,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.39790862798690796,
|
||
|
|
"rewards/smce": -0.026037893444299697,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1067.8,
|
||
|
|
"completions/max_terminated_length": 479.2,
|
||
|
|
"completions/mean_length": 141.0818359375,
|
||
|
|
"completions/mean_terminated_length": 140.6728057861328,
|
||
|
|
"completions/min_length": 62.6,
|
||
|
|
"completions/min_terminated_length": 62.6,
|
||
|
|
"epoch": 0.32,
|
||
|
|
"grad_norm": 0.0008635123958811164,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 336236632.0,
|
||
|
|
"reward": 1.0830219507217407,
|
||
|
|
"reward_std": 0.11141782402992248,
|
||
|
|
"rewards/accuracy_reward": 0.3953125,
|
||
|
|
"rewards/brier_reward": 0.7712236523628235,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0005859375,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4011562764644623,
|
||
|
|
"rewards/smce": -0.047462426498532294,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 387.5,
|
||
|
|
"eval_completions/max_terminated_length": 387.5,
|
||
|
|
"eval_completions/mean_length": 140.8268051147461,
|
||
|
|
"eval_completions/mean_terminated_length": 140.8268051147461,
|
||
|
|
"eval_completions/min_length": 72.5,
|
||
|
|
"eval_completions/min_terminated_length": 72.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 336236632.0,
|
||
|
|
"eval_reward": 1.0537142753601074,
|
||
|
|
"eval_reward_std": 0.20659654587507248,
|
||
|
|
"eval_rewards/accuracy_reward": 0.32421875,
|
||
|
|
"eval_rewards/brier_reward": 0.7832174003124237,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4059121161699295,
|
||
|
|
"eval_rewards/smce": -0.08268068730831146,
|
||
|
|
"eval_runtime": 12.3699,
|
||
|
|
"eval_samples_per_second": 40.421,
|
||
|
|
"eval_steps_per_second": 0.162,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1099.6,
|
||
|
|
"completions/max_terminated_length": 424.2,
|
||
|
|
"completions/mean_length": 141.2322265625,
|
||
|
|
"completions/mean_terminated_length": 140.82349853515626,
|
||
|
|
"completions/min_length": 64.4,
|
||
|
|
"completions/min_terminated_length": 64.4,
|
||
|
|
"epoch": 0.336,
|
||
|
|
"grad_norm": 0.0008358008344657719,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 352714338.0,
|
||
|
|
"reward": 1.0877865314483643,
|
||
|
|
"reward_std": 0.10996164083480835,
|
||
|
|
"rewards/accuracy_reward": 0.41025390625,
|
||
|
|
"rewards/brier_reward": 0.7656147480010986,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00087890625,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.40201926827430723,
|
||
|
|
"rewards/smce": -0.03372764438390732,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 597.6,
|
||
|
|
"completions/max_terminated_length": 354.6,
|
||
|
|
"completions/mean_length": 139.06103515625,
|
||
|
|
"completions/mean_terminated_length": 138.92465209960938,
|
||
|
|
"completions/min_length": 64.2,
|
||
|
|
"completions/min_terminated_length": 64.2,
|
||
|
|
"epoch": 0.352,
|
||
|
|
"grad_norm": 0.0008557201945222914,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 369174259.0,
|
||
|
|
"reward": 1.0942952394485475,
|
||
|
|
"reward_std": 0.11433759331703186,
|
||
|
|
"rewards/accuracy_reward": 0.41708984375,
|
||
|
|
"rewards/brier_reward": 0.7716007232666016,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.39787405729293823,
|
||
|
|
"rewards/smce": -0.032202685810625555,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 326.8,
|
||
|
|
"completions/max_terminated_length": 326.8,
|
||
|
|
"completions/mean_length": 138.03349609375,
|
||
|
|
"completions/mean_terminated_length": 138.03349609375,
|
||
|
|
"completions/min_length": 65.0,
|
||
|
|
"completions/min_terminated_length": 65.0,
|
||
|
|
"epoch": 0.368,
|
||
|
|
"grad_norm": 0.0008347496623173356,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 385609770.0,
|
||
|
|
"reward": 1.1024059534072876,
|
||
|
|
"reward_std": 0.10553737431764602,
|
||
|
|
"rewards/accuracy_reward": 0.42998046875,
|
||
|
|
"rewards/brier_reward": 0.7749350905418396,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.40569075345993044,
|
||
|
|
"rewards/smce": -0.06787904426455497,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 373.2,
|
||
|
|
"completions/max_terminated_length": 373.2,
|
||
|
|
"completions/mean_length": 142.7939453125,
|
||
|
|
"completions/mean_terminated_length": 142.7939453125,
|
||
|
|
"completions/min_length": 65.4,
|
||
|
|
"completions/min_terminated_length": 65.4,
|
||
|
|
"epoch": 0.384,
|
||
|
|
"grad_norm": 0.0008544361335225403,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 402015820.0,
|
||
|
|
"reward": 1.083213472366333,
|
||
|
|
"reward_std": 0.10497440248727799,
|
||
|
|
"rewards/accuracy_reward": 0.38974609375,
|
||
|
|
"rewards/brier_reward": 0.7766830205917359,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4167145550251007,
|
||
|
|
"rewards/smce": -0.030062619410455228,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 588.6,
|
||
|
|
"completions/max_terminated_length": 357.6,
|
||
|
|
"completions/mean_length": 144.31806640625,
|
||
|
|
"completions/mean_terminated_length": 144.18206787109375,
|
||
|
|
"completions/min_length": 65.4,
|
||
|
|
"completions/min_terminated_length": 65.4,
|
||
|
|
"epoch": 0.4,
|
||
|
|
"grad_norm": 0.001108017168007791,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 418605509.0,
|
||
|
|
"reward": 1.0979632616043091,
|
||
|
|
"reward_std": 0.10889875888824463,
|
||
|
|
"rewards/accuracy_reward": 0.42548828125,
|
||
|
|
"rewards/brier_reward": 0.7705417275428772,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4027201533317566,
|
||
|
|
"rewards/smce": -0.06595741771161556,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 836.2,
|
||
|
|
"completions/max_terminated_length": 363.0,
|
||
|
|
"completions/mean_length": 147.25498046875,
|
||
|
|
"completions/mean_terminated_length": 146.84811401367188,
|
||
|
|
"completions/min_length": 57.4,
|
||
|
|
"completions/min_terminated_length": 57.4,
|
||
|
|
"epoch": 0.416,
|
||
|
|
"grad_norm": 0.0009119771420955658,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 435183704.0,
|
||
|
|
"reward": 1.1149553060531616,
|
||
|
|
"reward_std": 0.1131348431110382,
|
||
|
|
"rewards/accuracy_reward": 0.4630859375,
|
||
|
|
"rewards/brier_reward": 0.767220401763916,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00068359375,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4106293976306915,
|
||
|
|
"rewards/smce": -0.05815782323479653,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 396.6,
|
||
|
|
"completions/max_terminated_length": 396.6,
|
||
|
|
"completions/mean_length": 148.89404296875,
|
||
|
|
"completions/mean_terminated_length": 148.89404296875,
|
||
|
|
"completions/min_length": 71.6,
|
||
|
|
"completions/min_terminated_length": 71.6,
|
||
|
|
"epoch": 0.432,
|
||
|
|
"grad_norm": 0.0009073733817785978,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 451666715.0,
|
||
|
|
"reward": 1.109913206100464,
|
||
|
|
"reward_std": 0.1156215637922287,
|
||
|
|
"rewards/accuracy_reward": 0.45029296875,
|
||
|
|
"rewards/brier_reward": 0.7695358991622925,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.44832324981689453,
|
||
|
|
"rewards/smce": -0.03289064317941666,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 589.2,
|
||
|
|
"completions/max_terminated_length": 375.2,
|
||
|
|
"completions/mean_length": 151.27353515625,
|
||
|
|
"completions/mean_terminated_length": 151.00367126464843,
|
||
|
|
"completions/min_length": 71.8,
|
||
|
|
"completions/min_terminated_length": 71.8,
|
||
|
|
"epoch": 0.448,
|
||
|
|
"grad_norm": 0.0008107796311378479,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 468305292.0,
|
||
|
|
"reward": 1.0973914384841919,
|
||
|
|
"reward_std": 0.11089750230312348,
|
||
|
|
"rewards/accuracy_reward": 0.42548828125,
|
||
|
|
"rewards/brier_reward": 0.7694926500320435,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.45723145008087157,
|
||
|
|
"rewards/smce": -0.03543643653392792,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 786.6,
|
||
|
|
"completions/max_terminated_length": 562.6,
|
||
|
|
"completions/mean_length": 154.874609375,
|
||
|
|
"completions/mean_terminated_length": 154.73968811035155,
|
||
|
|
"completions/min_length": 71.8,
|
||
|
|
"completions/min_terminated_length": 71.8,
|
||
|
|
"epoch": 0.464,
|
||
|
|
"grad_norm": 0.0008580603753216565,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 485007272.0,
|
||
|
|
"reward": 1.1149845123291016,
|
||
|
|
"reward_std": 0.1036541149020195,
|
||
|
|
"rewards/accuracy_reward": 0.44921875,
|
||
|
|
"rewards/brier_reward": 0.7808510184288024,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.45266503691673277,
|
||
|
|
"rewards/smce": -0.03915540650486946,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 589.0,
|
||
|
|
"completions/max_terminated_length": 355.6,
|
||
|
|
"completions/mean_length": 154.5427734375,
|
||
|
|
"completions/mean_terminated_length": 154.27273559570312,
|
||
|
|
"completions/min_length": 75.6,
|
||
|
|
"completions/min_terminated_length": 75.6,
|
||
|
|
"epoch": 0.48,
|
||
|
|
"grad_norm": 0.0008421405800618231,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 501708158.0,
|
||
|
|
"reward": 1.1118209838867188,
|
||
|
|
"reward_std": 0.11529819369316101,
|
||
|
|
"rewards/accuracy_reward": 0.44970703125,
|
||
|
|
"rewards/brier_reward": 0.7741313695907592,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.44507910013198854,
|
||
|
|
"rewards/smce": -0.019151285663247107,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.48,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 330.0,
|
||
|
|
"eval_completions/max_terminated_length": 330.0,
|
||
|
|
"eval_completions/mean_length": 156.71817016601562,
|
||
|
|
"eval_completions/mean_terminated_length": 156.71817016601562,
|
||
|
|
"eval_completions/min_length": 87.5,
|
||
|
|
"eval_completions/min_terminated_length": 87.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 501708158.0,
|
||
|
|
"eval_reward": 1.0748705863952637,
|
||
|
|
"eval_reward_std": 0.22966566681861877,
|
||
|
|
"eval_rewards/accuracy_reward": 0.36328125,
|
||
|
|
"eval_rewards/brier_reward": 0.7864670157432556,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4421289265155792,
|
||
|
|
"eval_rewards/smce": -0.07997188344597816,
|
||
|
|
"eval_runtime": 11.4907,
|
||
|
|
"eval_samples_per_second": 43.513,
|
||
|
|
"eval_steps_per_second": 0.174,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 873.4,
|
||
|
|
"completions/max_terminated_length": 420.2,
|
||
|
|
"completions/mean_length": 159.5671875,
|
||
|
|
"completions/mean_terminated_length": 159.2984588623047,
|
||
|
|
"completions/min_length": 73.0,
|
||
|
|
"completions/min_terminated_length": 73.0,
|
||
|
|
"epoch": 0.496,
|
||
|
|
"grad_norm": 0.000909183407202363,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 518364942.0,
|
||
|
|
"reward": 1.084216022491455,
|
||
|
|
"reward_std": 0.10733449161052704,
|
||
|
|
"rewards/accuracy_reward": 0.3853515625,
|
||
|
|
"rewards/brier_reward": 0.7832805037498474,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.43932910561561583,
|
||
|
|
"rewards/smce": -0.05489234179258347,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 1111.0,
|
||
|
|
"completions/max_terminated_length": 430.6,
|
||
|
|
"completions/mean_length": 159.97548828125,
|
||
|
|
"completions/mean_terminated_length": 159.43768005371095,
|
||
|
|
"completions/min_length": 71.6,
|
||
|
|
"completions/min_terminated_length": 71.6,
|
||
|
|
"epoch": 0.512,
|
||
|
|
"grad_norm": 0.0007752343663014472,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"num_tokens": 534732563.0,
|
||
|
|
"reward": 1.107197880744934,
|
||
|
|
"reward_std": 0.1062451258301735,
|
||
|
|
"rewards/accuracy_reward": 0.43896484375,
|
||
|
|
"rewards/brier_reward": 0.7758240461349487,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4436230480670929,
|
||
|
|
"rewards/smce": -0.03356703966856003,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 649.6,
|
||
|
|
"completions/max_terminated_length": 410.4,
|
||
|
|
"completions/mean_length": 161.52412109375,
|
||
|
|
"completions/mean_terminated_length": 161.38986206054688,
|
||
|
|
"completions/min_length": 79.6,
|
||
|
|
"completions/min_terminated_length": 79.6,
|
||
|
|
"epoch": 0.528,
|
||
|
|
"grad_norm": 0.000873538083396852,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 551442378.0,
|
||
|
|
"reward": 1.1122208118438721,
|
||
|
|
"reward_std": 0.10901667475700379,
|
||
|
|
"rewards/accuracy_reward": 0.45302734375,
|
||
|
|
"rewards/brier_reward": 0.771516752243042,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0005859375,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4236679673194885,
|
||
|
|
"rewards/smce": -0.055811258032917976,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 663.8,
|
||
|
|
"completions/max_terminated_length": 451.6,
|
||
|
|
"completions/mean_length": 162.53212890625,
|
||
|
|
"completions/mean_terminated_length": 162.3982421875,
|
||
|
|
"completions/min_length": 79.8,
|
||
|
|
"completions/min_terminated_length": 79.8,
|
||
|
|
"epoch": 0.544,
|
||
|
|
"grad_norm": 0.0008957363897934556,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 568087443.0,
|
||
|
|
"reward": 1.1219069719314576,
|
||
|
|
"reward_std": 0.10789882093667984,
|
||
|
|
"rewards/accuracy_reward": 0.46884765625,
|
||
|
|
"rewards/brier_reward": 0.7751662611961365,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4372187554836273,
|
||
|
|
"rewards/smce": -0.05467262491583824,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 606.2,
|
||
|
|
"completions/max_terminated_length": 382.6,
|
||
|
|
"completions/mean_length": 159.515625,
|
||
|
|
"completions/mean_terminated_length": 159.3811279296875,
|
||
|
|
"completions/min_length": 77.6,
|
||
|
|
"completions/min_terminated_length": 77.6,
|
||
|
|
"epoch": 0.56,
|
||
|
|
"grad_norm": 0.000858804676681757,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"num_tokens": 584687539.0,
|
||
|
|
"reward": 1.1200573682785033,
|
||
|
|
"reward_std": 0.10227487087249756,
|
||
|
|
"rewards/accuracy_reward": 0.45849609375,
|
||
|
|
"rewards/brier_reward": 0.7817179918289184,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4568593859672546,
|
||
|
|
"rewards/smce": -0.026216423511505126,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 446.8,
|
||
|
|
"completions/max_terminated_length": 446.8,
|
||
|
|
"completions/mean_length": 158.66416015625,
|
||
|
|
"completions/mean_terminated_length": 158.66416015625,
|
||
|
|
"completions/min_length": 75.0,
|
||
|
|
"completions/min_terminated_length": 75.0,
|
||
|
|
"epoch": 0.576,
|
||
|
|
"grad_norm": 0.0010204884456470609,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 601433508.0,
|
||
|
|
"reward": 1.1217976331710815,
|
||
|
|
"reward_std": 0.1035456970334053,
|
||
|
|
"rewards/accuracy_reward": 0.44609375,
|
||
|
|
"rewards/brier_reward": 0.7975054860115052,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.43276728987693786,
|
||
|
|
"rewards/smce": -0.0471338476985693,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 431.2,
|
||
|
|
"completions/max_terminated_length": 431.2,
|
||
|
|
"completions/mean_length": 159.45615234375,
|
||
|
|
"completions/mean_terminated_length": 159.45615234375,
|
||
|
|
"completions/min_length": 81.8,
|
||
|
|
"completions/min_terminated_length": 81.8,
|
||
|
|
"epoch": 0.592,
|
||
|
|
"grad_norm": 0.0008188558858819306,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 617936419.0,
|
||
|
|
"reward": 1.1226669073104858,
|
||
|
|
"reward_std": 0.10481721758842469,
|
||
|
|
"rewards/accuracy_reward": 0.45576171875,
|
||
|
|
"rewards/brier_reward": 0.7895737171173096,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.44277440905570986,
|
||
|
|
"rewards/smce": -0.024372975155711175,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 863.6,
|
||
|
|
"completions/max_terminated_length": 428.8,
|
||
|
|
"completions/mean_length": 164.56435546875,
|
||
|
|
"completions/mean_terminated_length": 164.2962646484375,
|
||
|
|
"completions/min_length": 80.0,
|
||
|
|
"completions/min_terminated_length": 80.0,
|
||
|
|
"epoch": 0.608,
|
||
|
|
"grad_norm": 0.0008857807843014598,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 634850998.0,
|
||
|
|
"reward": 1.1132190942764282,
|
||
|
|
"reward_std": 0.11314407587051392,
|
||
|
|
"rewards/accuracy_reward": 0.437109375,
|
||
|
|
"rewards/brier_reward": 0.7895278453826904,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4769873142242432,
|
||
|
|
"rewards/smce": -0.04614109750837088,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 655.4,
|
||
|
|
"completions/max_terminated_length": 440.0,
|
||
|
|
"completions/mean_length": 161.4076171875,
|
||
|
|
"completions/mean_terminated_length": 161.27352600097657,
|
||
|
|
"completions/min_length": 82.8,
|
||
|
|
"completions/min_terminated_length": 82.8,
|
||
|
|
"epoch": 0.624,
|
||
|
|
"grad_norm": 0.0009903701720759273,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 651472612.0,
|
||
|
|
"reward": 1.1419665813446045,
|
||
|
|
"reward_std": 0.10531192421913146,
|
||
|
|
"rewards/accuracy_reward": 0.49169921875,
|
||
|
|
"rewards/brier_reward": 0.7923331260681152,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.48211621642112734,
|
||
|
|
"rewards/smce": -0.024501551128923892,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 675.2,
|
||
|
|
"completions/max_terminated_length": 435.4,
|
||
|
|
"completions/mean_length": 158.34560546875,
|
||
|
|
"completions/mean_terminated_length": 158.0752716064453,
|
||
|
|
"completions/min_length": 80.6,
|
||
|
|
"completions/min_terminated_length": 80.6,
|
||
|
|
"epoch": 0.64,
|
||
|
|
"grad_norm": 0.0009547388763166964,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 668172215.0,
|
||
|
|
"reward": 1.1136085271835328,
|
||
|
|
"reward_std": 0.10880862027406693,
|
||
|
|
"rewards/accuracy_reward": 0.43544921875,
|
||
|
|
"rewards/brier_reward": 0.7919665455818177,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4641181766986847,
|
||
|
|
"rewards/smce": -0.042938829958438875,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.64,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 319.0,
|
||
|
|
"eval_completions/max_terminated_length": 319.0,
|
||
|
|
"eval_completions/mean_length": 158.68778228759766,
|
||
|
|
"eval_completions/mean_terminated_length": 158.68778228759766,
|
||
|
|
"eval_completions/min_length": 87.5,
|
||
|
|
"eval_completions/min_terminated_length": 87.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 668172215.0,
|
||
|
|
"eval_reward": 1.0859458446502686,
|
||
|
|
"eval_reward_std": 0.25144463777542114,
|
||
|
|
"eval_rewards/accuracy_reward": 0.384765625,
|
||
|
|
"eval_rewards/brier_reward": 0.7871337831020355,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4686523377895355,
|
||
|
|
"eval_rewards/smce": -0.08460449427366257,
|
||
|
|
"eval_runtime": 11.2462,
|
||
|
|
"eval_samples_per_second": 44.46,
|
||
|
|
"eval_steps_per_second": 0.178,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 380.4,
|
||
|
|
"completions/max_terminated_length": 380.4,
|
||
|
|
"completions/mean_length": 158.65654296875,
|
||
|
|
"completions/mean_terminated_length": 158.65654296875,
|
||
|
|
"completions/min_length": 79.8,
|
||
|
|
"completions/min_terminated_length": 79.8,
|
||
|
|
"epoch": 0.656,
|
||
|
|
"grad_norm": 0.0008885656134225428,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 684988570.0,
|
||
|
|
"reward": 1.124780035018921,
|
||
|
|
"reward_std": 0.11241506040096283,
|
||
|
|
"rewards/accuracy_reward": 0.46171875,
|
||
|
|
"rewards/brier_reward": 0.787843644618988,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4735654413700104,
|
||
|
|
"rewards/smce": -0.032851331681013105,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 679.4,
|
||
|
|
"completions/max_terminated_length": 461.4,
|
||
|
|
"completions/mean_length": 160.13818359375,
|
||
|
|
"completions/mean_terminated_length": 160.00401000976564,
|
||
|
|
"completions/min_length": 81.2,
|
||
|
|
"completions/min_terminated_length": 81.2,
|
||
|
|
"epoch": 0.672,
|
||
|
|
"grad_norm": 0.0009205000242218375,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 701576577.0,
|
||
|
|
"reward": 1.1205364227294923,
|
||
|
|
"reward_std": 0.10743532329797745,
|
||
|
|
"rewards/accuracy_reward": 0.4419921875,
|
||
|
|
"rewards/brier_reward": 0.7992789030075074,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.47032617330551146,
|
||
|
|
"rewards/smce": -0.03808985371142626,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 880.6,
|
||
|
|
"completions/max_terminated_length": 419.4,
|
||
|
|
"completions/mean_length": 159.65302734375,
|
||
|
|
"completions/mean_terminated_length": 159.384033203125,
|
||
|
|
"completions/min_length": 84.2,
|
||
|
|
"completions/min_terminated_length": 84.2,
|
||
|
|
"epoch": 0.688,
|
||
|
|
"grad_norm": 0.0011366148246452212,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 718239936.0,
|
||
|
|
"reward": 1.1395163536071777,
|
||
|
|
"reward_std": 0.10088872164487839,
|
||
|
|
"rewards/accuracy_reward": 0.48984375,
|
||
|
|
"rewards/brier_reward": 0.7893894791603089,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.43761621713638305,
|
||
|
|
"rewards/smce": -0.06050069257616997,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 639.4,
|
||
|
|
"completions/max_terminated_length": 403.6,
|
||
|
|
"completions/mean_length": 163.80947265625,
|
||
|
|
"completions/mean_terminated_length": 163.67565002441407,
|
||
|
|
"completions/min_length": 82.0,
|
||
|
|
"completions/min_terminated_length": 82.0,
|
||
|
|
"epoch": 0.704,
|
||
|
|
"grad_norm": 0.0008274089777842164,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 735125121.0,
|
||
|
|
"reward": 1.1324799537658692,
|
||
|
|
"reward_std": 0.10014449208974838,
|
||
|
|
"rewards/accuracy_reward": 0.48134765625,
|
||
|
|
"rewards/brier_reward": 0.7837143540382385,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.42956447005271914,
|
||
|
|
"rewards/smce": -0.052855022624135016,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 614.2,
|
||
|
|
"completions/max_terminated_length": 386.2,
|
||
|
|
"completions/mean_length": 165.41416015625,
|
||
|
|
"completions/mean_terminated_length": 165.2800323486328,
|
||
|
|
"completions/min_length": 86.6,
|
||
|
|
"completions/min_terminated_length": 86.6,
|
||
|
|
"epoch": 0.72,
|
||
|
|
"grad_norm": 0.001209041103720665,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 751689842.0,
|
||
|
|
"reward": 1.1413109302520752,
|
||
|
|
"reward_std": 0.10785426497459412,
|
||
|
|
"rewards/accuracy_reward": 0.48271484375,
|
||
|
|
"rewards/brier_reward": 0.800009298324585,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4845683515071869,
|
||
|
|
"rewards/smce": -0.05454383902251721,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 887.4,
|
||
|
|
"completions/max_terminated_length": 443.6,
|
||
|
|
"completions/mean_length": 167.75576171875,
|
||
|
|
"completions/mean_terminated_length": 167.4881164550781,
|
||
|
|
"completions/min_length": 84.8,
|
||
|
|
"completions/min_terminated_length": 84.8,
|
||
|
|
"epoch": 0.736,
|
||
|
|
"grad_norm": 0.0009000345598906279,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 768583053.0,
|
||
|
|
"reward": 1.1350015878677369,
|
||
|
|
"reward_std": 0.11322512924671173,
|
||
|
|
"rewards/accuracy_reward": 0.47939453125,
|
||
|
|
"rewards/brier_reward": 0.7908076047897339,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5125478506088257,
|
||
|
|
"rewards/smce": -0.046359357237815854,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 706.8,
|
||
|
|
"completions/max_terminated_length": 509.0,
|
||
|
|
"completions/mean_length": 171.1728515625,
|
||
|
|
"completions/mean_terminated_length": 171.04000854492188,
|
||
|
|
"completions/min_length": 92.4,
|
||
|
|
"completions/min_terminated_length": 92.4,
|
||
|
|
"epoch": 0.752,
|
||
|
|
"grad_norm": 0.000924249179661274,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 785295447.0,
|
||
|
|
"reward": 1.1308918714523315,
|
||
|
|
"reward_std": 0.10929270833730698,
|
||
|
|
"rewards/accuracy_reward": 0.47255859375,
|
||
|
|
"rewards/brier_reward": 0.7893296003341674,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4867382884025574,
|
||
|
|
"rewards/smce": -0.07814770340919494,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 629.2,
|
||
|
|
"completions/max_terminated_length": 629.2,
|
||
|
|
"completions/mean_length": 174.9046875,
|
||
|
|
"completions/mean_terminated_length": 174.9046875,
|
||
|
|
"completions/min_length": 91.6,
|
||
|
|
"completions/min_terminated_length": 91.6,
|
||
|
|
"epoch": 0.768,
|
||
|
|
"grad_norm": 0.0008467729785479605,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 802093895.0,
|
||
|
|
"reward": 1.1407481908798218,
|
||
|
|
"reward_std": 0.09571124464273453,
|
||
|
|
"rewards/accuracy_reward": 0.473828125,
|
||
|
|
"rewards/brier_reward": 0.8076701879501342,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4546728551387787,
|
||
|
|
"rewards/smce": -0.028032467886805534,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 702.6,
|
||
|
|
"completions/max_terminated_length": 482.4,
|
||
|
|
"completions/mean_length": 173.15537109375,
|
||
|
|
"completions/mean_terminated_length": 173.02240295410155,
|
||
|
|
"completions/min_length": 89.6,
|
||
|
|
"completions/min_terminated_length": 89.6,
|
||
|
|
"epoch": 0.784,
|
||
|
|
"grad_norm": 0.00092527293600142,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 818869630.0,
|
||
|
|
"reward": 1.1412487745285034,
|
||
|
|
"reward_std": 0.10290929973125458,
|
||
|
|
"rewards/accuracy_reward": 0.47958984375,
|
||
|
|
"rewards/brier_reward": 0.8030083179473877,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4737207114696503,
|
||
|
|
"rewards/smce": -0.03872436471283436,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 884.0,
|
||
|
|
"completions/max_terminated_length": 459.4,
|
||
|
|
"completions/mean_length": 177.40078125,
|
||
|
|
"completions/mean_terminated_length": 177.0030487060547,
|
||
|
|
"completions/min_length": 94.8,
|
||
|
|
"completions/min_terminated_length": 94.8,
|
||
|
|
"epoch": 0.8,
|
||
|
|
"grad_norm": 0.001135337632149458,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 835906438.0,
|
||
|
|
"reward": 1.142169713973999,
|
||
|
|
"reward_std": 0.11441078335046768,
|
||
|
|
"rewards/accuracy_reward": 0.48525390625,
|
||
|
|
"rewards/brier_reward": 0.7993808865547181,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4921103477478027,
|
||
|
|
"rewards/smce": -0.03339071050286293,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 346.0,
|
||
|
|
"eval_completions/max_terminated_length": 346.0,
|
||
|
|
"eval_completions/mean_length": 185.98580932617188,
|
||
|
|
"eval_completions/mean_terminated_length": 185.98580932617188,
|
||
|
|
"eval_completions/min_length": 103.5,
|
||
|
|
"eval_completions/min_terminated_length": 103.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 835906438.0,
|
||
|
|
"eval_reward": 1.0858368873596191,
|
||
|
|
"eval_reward_std": 0.2645293176174164,
|
||
|
|
"eval_rewards/accuracy_reward": 0.375,
|
||
|
|
"eval_rewards/brier_reward": 0.7966845631599426,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4926757961511612,
|
||
|
|
"eval_rewards/smce": -0.1178906224668026,
|
||
|
|
"eval_runtime": 11.9011,
|
||
|
|
"eval_samples_per_second": 42.013,
|
||
|
|
"eval_steps_per_second": 0.168,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 698.0,
|
||
|
|
"completions/max_terminated_length": 469.0,
|
||
|
|
"completions/mean_length": 186.46650390625,
|
||
|
|
"completions/mean_terminated_length": 186.33453979492188,
|
||
|
|
"completions/min_length": 99.8,
|
||
|
|
"completions/min_terminated_length": 99.8,
|
||
|
|
"epoch": 0.816,
|
||
|
|
"grad_norm": 0.0008341626380570233,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 852905775.0,
|
||
|
|
"reward": 1.1321615695953369,
|
||
|
|
"reward_std": 0.09045738875865936,
|
||
|
|
"rewards/accuracy_reward": 0.46025390625,
|
||
|
|
"rewards/brier_reward": 0.804168701171875,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.47615723609924315,
|
||
|
|
"rewards/smce": -0.026564698852598666,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 665.6,
|
||
|
|
"completions/max_terminated_length": 482.6,
|
||
|
|
"completions/mean_length": 186.53017578125,
|
||
|
|
"completions/mean_terminated_length": 186.39850463867188,
|
||
|
|
"completions/min_length": 97.2,
|
||
|
|
"completions/min_terminated_length": 97.2,
|
||
|
|
"epoch": 0.832,
|
||
|
|
"grad_norm": 0.0010087019763886929,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 869908228.0,
|
||
|
|
"reward": 1.1526384592056274,
|
||
|
|
"reward_std": 0.10760405957698822,
|
||
|
|
"rewards/accuracy_reward": 0.50556640625,
|
||
|
|
"rewards/brier_reward": 0.7998112916946412,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4938330054283142,
|
||
|
|
"rewards/smce": -0.040554934181272984,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 671.8,
|
||
|
|
"completions/max_terminated_length": 461.6,
|
||
|
|
"completions/mean_length": 186.29765625,
|
||
|
|
"completions/mean_terminated_length": 186.16553649902343,
|
||
|
|
"completions/min_length": 96.4,
|
||
|
|
"completions/min_terminated_length": 96.4,
|
||
|
|
"epoch": 0.848,
|
||
|
|
"grad_norm": 0.0010384187335148454,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 886850124.0,
|
||
|
|
"reward": 1.1427557468414307,
|
||
|
|
"reward_std": 0.10271692872047425,
|
||
|
|
"rewards/accuracy_reward": 0.4865234375,
|
||
|
|
"rewards/brier_reward": 0.7990883827209473,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.47125096917152404,
|
||
|
|
"rewards/smce": -0.03518822770565748,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 659.0,
|
||
|
|
"completions/max_terminated_length": 433.6,
|
||
|
|
"completions/mean_length": 180.399609375,
|
||
|
|
"completions/mean_terminated_length": 180.26744384765624,
|
||
|
|
"completions/min_length": 87.6,
|
||
|
|
"completions/min_terminated_length": 87.6,
|
||
|
|
"epoch": 0.864,
|
||
|
|
"grad_norm": 0.0007400948088616133,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 903635656.0,
|
||
|
|
"reward": 1.136153483390808,
|
||
|
|
"reward_std": 0.08060802072286606,
|
||
|
|
"rewards/accuracy_reward": 0.458203125,
|
||
|
|
"rewards/brier_reward": 0.8143020153045655,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4435312509536743,
|
||
|
|
"rewards/smce": -0.03686932548880577,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 652.2,
|
||
|
|
"completions/max_terminated_length": 440.2,
|
||
|
|
"completions/mean_length": 182.7515625,
|
||
|
|
"completions/mean_terminated_length": 182.48705139160157,
|
||
|
|
"completions/min_length": 97.6,
|
||
|
|
"completions/min_terminated_length": 97.6,
|
||
|
|
"epoch": 0.88,
|
||
|
|
"grad_norm": 0.0008531102794222534,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 920581784.0,
|
||
|
|
"reward": 1.1429947853088378,
|
||
|
|
"reward_std": 0.09263245761394501,
|
||
|
|
"rewards/accuracy_reward": 0.4791015625,
|
||
|
|
"rewards/brier_reward": 0.8071859121322632,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.44282519817352295,
|
||
|
|
"rewards/smce": -0.0567308247089386,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 634.0,
|
||
|
|
"completions/max_terminated_length": 419.2,
|
||
|
|
"completions/mean_length": 182.8693359375,
|
||
|
|
"completions/mean_terminated_length": 182.73717346191407,
|
||
|
|
"completions/min_length": 94.2,
|
||
|
|
"completions/min_terminated_length": 94.2,
|
||
|
|
"epoch": 0.896,
|
||
|
|
"grad_norm": 0.0008840742520987988,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 937587198.0,
|
||
|
|
"reward": 1.1388145685195923,
|
||
|
|
"reward_std": 0.09899315983057022,
|
||
|
|
"rewards/accuracy_reward": 0.48037109375,
|
||
|
|
"rewards/brier_reward": 0.7974554300308228,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.49169726967811583,
|
||
|
|
"rewards/smce": -0.02959427684545517,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 690.8,
|
||
|
|
"completions/max_terminated_length": 477.0,
|
||
|
|
"completions/mean_length": 182.4890625,
|
||
|
|
"completions/mean_terminated_length": 182.35704650878907,
|
||
|
|
"completions/min_length": 92.6,
|
||
|
|
"completions/min_terminated_length": 92.6,
|
||
|
|
"epoch": 0.912,
|
||
|
|
"grad_norm": 0.0008525560842826962,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 954336110.0,
|
||
|
|
"reward": 1.138223648071289,
|
||
|
|
"reward_std": 0.10281311720609665,
|
||
|
|
"rewards/accuracy_reward": 0.4755859375,
|
||
|
|
"rewards/brier_reward": 0.8010599613189697,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.49716113805770873,
|
||
|
|
"rewards/smce": -0.04195708781480789,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 869.8,
|
||
|
|
"completions/max_terminated_length": 438.0,
|
||
|
|
"completions/mean_length": 181.30810546875,
|
||
|
|
"completions/mean_terminated_length": 181.0434326171875,
|
||
|
|
"completions/min_length": 97.0,
|
||
|
|
"completions/min_terminated_length": 97.0,
|
||
|
|
"epoch": 0.928,
|
||
|
|
"grad_norm": 0.0010722881415858865,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 971141569.0,
|
||
|
|
"reward": 1.1697998523712159,
|
||
|
|
"reward_std": 0.09429487735033035,
|
||
|
|
"rewards/accuracy_reward": 0.5265625,
|
||
|
|
"rewards/brier_reward": 0.8132358431816101,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5027734339237213,
|
||
|
|
"rewards/smce": -0.04298241809010506,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1309.6,
|
||
|
|
"completions/max_terminated_length": 562.4,
|
||
|
|
"completions/mean_length": 187.14462890625,
|
||
|
|
"completions/mean_terminated_length": 186.48440856933593,
|
||
|
|
"completions/min_length": 99.2,
|
||
|
|
"completions/min_terminated_length": 99.2,
|
||
|
|
"epoch": 0.944,
|
||
|
|
"grad_norm": 0.000979841803200543,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"num_tokens": 988275658.0,
|
||
|
|
"reward": 1.1545918464660645,
|
||
|
|
"reward_std": 0.10092489123344421,
|
||
|
|
"rewards/accuracy_reward": 0.49013671875,
|
||
|
|
"rewards/brier_reward": 0.8196364879608155,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9994140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.49853025674819945,
|
||
|
|
"rewards/smce": -0.04494751356542111,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 599.4,
|
||
|
|
"completions/max_terminated_length": 599.4,
|
||
|
|
"completions/mean_length": 186.77529296875,
|
||
|
|
"completions/mean_terminated_length": 186.77529296875,
|
||
|
|
"completions/min_length": 98.2,
|
||
|
|
"completions/min_terminated_length": 98.2,
|
||
|
|
"epoch": 0.96,
|
||
|
|
"grad_norm": 0.001181174535304308,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 1005166701.0,
|
||
|
|
"reward": 1.1341146945953369,
|
||
|
|
"reward_std": 0.11333226412534714,
|
||
|
|
"rewards/accuracy_reward": 0.46708984375,
|
||
|
|
"rewards/brier_reward": 0.8011467099189759,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.5431005716323852,
|
||
|
|
"rewards/smce": -0.08158175013959408,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.96,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 577.5,
|
||
|
|
"eval_completions/max_terminated_length": 577.5,
|
||
|
|
"eval_completions/mean_length": 191.35697174072266,
|
||
|
|
"eval_completions/mean_terminated_length": 191.35697174072266,
|
||
|
|
"eval_completions/min_length": 109.5,
|
||
|
|
"eval_completions/min_terminated_length": 109.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 1005166701.0,
|
||
|
|
"eval_reward": 1.0900059342384338,
|
||
|
|
"eval_reward_std": 0.2852499932050705,
|
||
|
|
"eval_rewards/accuracy_reward": 0.380859375,
|
||
|
|
"eval_rewards/brier_reward": 0.7991650402545929,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.5155273675918579,
|
||
|
|
"eval_rewards/smce": -0.13530762493610382,
|
||
|
|
"eval_runtime": 15.8845,
|
||
|
|
"eval_samples_per_second": 31.477,
|
||
|
|
"eval_steps_per_second": 0.126,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 586.8,
|
||
|
|
"completions/max_terminated_length": 586.8,
|
||
|
|
"completions/mean_length": 189.49833984375,
|
||
|
|
"completions/mean_terminated_length": 189.49833984375,
|
||
|
|
"completions/min_length": 99.0,
|
||
|
|
"completions/min_terminated_length": 99.0,
|
||
|
|
"epoch": 0.976,
|
||
|
|
"grad_norm": 0.0009122914634644985,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 1022172476.0,
|
||
|
|
"reward": 1.151877188682556,
|
||
|
|
"reward_std": 0.09684720933437348,
|
||
|
|
"rewards/accuracy_reward": 0.49482421875,
|
||
|
|
"rewards/brier_reward": 0.8090285301208496,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.49103320837020875,
|
||
|
|
"rewards/smce": -0.016386817023158074,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 898.2,
|
||
|
|
"completions/max_terminated_length": 472.8,
|
||
|
|
"completions/mean_length": 196.90341796875,
|
||
|
|
"completions/mean_terminated_length": 196.51105041503905,
|
||
|
|
"completions/min_length": 99.0,
|
||
|
|
"completions/min_terminated_length": 99.0,
|
||
|
|
"epoch": 0.992,
|
||
|
|
"grad_norm": 0.0009238107013516128,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"num_tokens": 1039372479.0,
|
||
|
|
"reward": 1.168659234046936,
|
||
|
|
"reward_std": 0.08935579359531402,
|
||
|
|
"rewards/accuracy_reward": 0.534765625,
|
||
|
|
"rewards/brier_reward": 0.8028544902801513,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4400693416595459,
|
||
|
|
"rewards/smce": -0.09543538391590119,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 628.0,
|
||
|
|
"completions/max_terminated_length": 628.0,
|
||
|
|
"completions/mean_length": 202.2728042602539,
|
||
|
|
"completions/mean_terminated_length": 202.2728042602539,
|
||
|
|
"completions/min_length": 105.5,
|
||
|
|
"completions/min_terminated_length": 105.5,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"num_tokens": 1046240895.0,
|
||
|
|
"reward": 1.1533426642417908,
|
||
|
|
"reward_std": 0.09558231383562088,
|
||
|
|
"rewards/accuracy_reward": 0.49169921875,
|
||
|
|
"rewards/brier_reward": 0.8152325451374054,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.999755859375,
|
||
|
|
"rewards/mean_confidence_reward": 0.46422119438648224,
|
||
|
|
"rewards/smce": -0.03218200244009495,
|
||
|
|
"step": 312,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.004010988894338087,
|
||
|
|
"train_runtime": 42587.7875,
|
||
|
|
"train_samples_per_second": 0.47,
|
||
|
|
"train_steps_per_second": 0.007
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 312,
|
||
|
|
"num_input_tokens_seen": 1046240895,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 60,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 4,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|