1944 lines
72 KiB
JSON
1944 lines
72 KiB
JSON
{
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.9984,
|
|
"eval_steps": 50,
|
|
"global_step": 312,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.11337890625,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1526.8,
|
|
"completions/mean_length": 487.7326171875,
|
|
"completions/mean_terminated_length": 353.7248291015625,
|
|
"completions/min_length": 1.0,
|
|
"completions/min_terminated_length": 1.0,
|
|
"epoch": 0.016,
|
|
"grad_norm": 0.0035850778222084045,
|
|
"learning_rate": 3.1249999999999997e-07,
|
|
"loss": 0.0442,
|
|
"num_tokens": 19980846.0,
|
|
"reward": 0.20237886905670166,
|
|
"reward_std": 0.3675601720809937,
|
|
"rewards/accuracy_reward": 0.05322265625,
|
|
"rewards/brier_reward": 0.09767542034387589,
|
|
"rewards/confidence_one_or_zero": 0.41123046875,
|
|
"rewards/format_reward": 0.25390625,
|
|
"rewards/mean_confidence_reward": 0.5377732753753662,
|
|
"rewards/smce": -0.484735369682312,
|
|
"step": 5
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1056640625,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1527.4,
|
|
"completions/mean_length": 448.823046875,
|
|
"completions/mean_terminated_length": 320.8543334960938,
|
|
"completions/min_length": 1.0,
|
|
"completions/min_terminated_length": 1.0,
|
|
"epoch": 0.032,
|
|
"grad_norm": 0.004688555840402842,
|
|
"learning_rate": 6.249999999999999e-07,
|
|
"loss": 0.045,
|
|
"num_tokens": 39560218.0,
|
|
"reward": 0.2146211951971054,
|
|
"reward_std": 0.37262771725654603,
|
|
"rewards/accuracy_reward": 0.05390625,
|
|
"rewards/brier_reward": 0.10282630771398545,
|
|
"rewards/confidence_one_or_zero": 0.435546875,
|
|
"rewards/format_reward": 0.27255859375,
|
|
"rewards/mean_confidence_reward": 0.5611979961395264,
|
|
"rewards/smce": -0.5075305879116059,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0509765625,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1487.4,
|
|
"completions/mean_length": 311.6556640625,
|
|
"completions/mean_terminated_length": 246.38412780761718,
|
|
"completions/min_length": 1.4,
|
|
"completions/min_terminated_length": 1.4,
|
|
"epoch": 0.048,
|
|
"grad_norm": 0.004514631815254688,
|
|
"learning_rate": 9.374999999999999e-07,
|
|
"loss": 0.0503,
|
|
"num_tokens": 57817620.0,
|
|
"reward": 0.40357725620269774,
|
|
"reward_std": 0.44316648244857787,
|
|
"rewards/accuracy_reward": 0.10810546875,
|
|
"rewards/brier_reward": 0.199105966091156,
|
|
"rewards/confidence_one_or_zero": 0.5349609375,
|
|
"rewards/format_reward": 0.5,
|
|
"rewards/mean_confidence_reward": 0.7012235999107361,
|
|
"rewards/smce": -0.5937792539596558,
|
|
"step": 15
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0224609375,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1454.8,
|
|
"completions/mean_length": 211.46875,
|
|
"completions/mean_terminated_length": 181.1057159423828,
|
|
"completions/min_length": 15.4,
|
|
"completions/min_terminated_length": 15.4,
|
|
"epoch": 0.064,
|
|
"grad_norm": 0.0024940520524978638,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0369,
|
|
"num_tokens": 75034132.0,
|
|
"reward": 0.6788389444351196,
|
|
"reward_std": 0.3938281416893005,
|
|
"rewards/accuracy_reward": 0.189453125,
|
|
"rewards/brier_reward": 0.3284399390220642,
|
|
"rewards/confidence_one_or_zero": 0.5779296875,
|
|
"rewards/format_reward": 0.83984375,
|
|
"rewards/mean_confidence_reward": 0.8039127707481384,
|
|
"rewards/smce": -0.6165119409561157,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.01513671875,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1392.2,
|
|
"completions/mean_length": 163.4111328125,
|
|
"completions/mean_terminated_length": 142.32491455078124,
|
|
"completions/min_length": 20.8,
|
|
"completions/min_terminated_length": 20.8,
|
|
"epoch": 0.08,
|
|
"grad_norm": 0.0027408404275774956,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0234,
|
|
"num_tokens": 91639238.0,
|
|
"reward": 0.8109780430793763,
|
|
"reward_std": 0.3374348938465118,
|
|
"rewards/accuracy_reward": 0.2431640625,
|
|
"rewards/brier_reward": 0.42249656319618223,
|
|
"rewards/confidence_one_or_zero": 0.50439453125,
|
|
"rewards/format_reward": 0.95634765625,
|
|
"rewards/mean_confidence_reward": 0.7859874844551087,
|
|
"rewards/smce": -0.547917366027832,
|
|
"step": 25
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0091796875,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1320.8,
|
|
"completions/mean_length": 145.26171875,
|
|
"completions/mean_terminated_length": 132.38874359130858,
|
|
"completions/min_length": 13.8,
|
|
"completions/min_terminated_length": 13.8,
|
|
"epoch": 0.096,
|
|
"grad_norm": 0.0023242132738232613,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0154,
|
|
"num_tokens": 107931710.0,
|
|
"reward": 0.8883284211158753,
|
|
"reward_std": 0.2937169373035431,
|
|
"rewards/accuracy_reward": 0.2525390625,
|
|
"rewards/brier_reward": 0.5444689631462097,
|
|
"rewards/confidence_one_or_zero": 0.3251953125,
|
|
"rewards/format_reward": 0.9796875,
|
|
"rewards/mean_confidence_reward": 0.6470721125602722,
|
|
"rewards/smce": -0.4061504006385803,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0044921875,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 866.8,
|
|
"completions/mean_length": 127.86181640625,
|
|
"completions/mean_terminated_length": 121.51152038574219,
|
|
"completions/min_length": 36.6,
|
|
"completions/min_terminated_length": 36.6,
|
|
"epoch": 0.112,
|
|
"grad_norm": 0.0018074740655720234,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0098,
|
|
"num_tokens": 124109815.0,
|
|
"reward": 0.9837763190269471,
|
|
"reward_std": 0.2189751923084259,
|
|
"rewards/accuracy_reward": 0.2767578125,
|
|
"rewards/brier_reward": 0.69852614402771,
|
|
"rewards/confidence_one_or_zero": 0.18828125,
|
|
"rewards/format_reward": 0.99228515625,
|
|
"rewards/mean_confidence_reward": 0.4274466037750244,
|
|
"rewards/smce": -0.17678792178630828,
|
|
"step": 35
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00244140625,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 691.0,
|
|
"completions/mean_length": 124.60478515625,
|
|
"completions/mean_terminated_length": 121.14984130859375,
|
|
"completions/min_length": 35.4,
|
|
"completions/min_terminated_length": 35.4,
|
|
"epoch": 0.128,
|
|
"grad_norm": 0.0009728963486850262,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0049,
|
|
"num_tokens": 140367656.0,
|
|
"reward": 1.0210827350616456,
|
|
"reward_std": 0.15196589678525924,
|
|
"rewards/accuracy_reward": 0.28193359375,
|
|
"rewards/brier_reward": 0.7644367456436157,
|
|
"rewards/confidence_one_or_zero": 0.146875,
|
|
"rewards/format_reward": 0.99580078125,
|
|
"rewards/mean_confidence_reward": 0.28299830853939056,
|
|
"rewards/smce": -0.06476258486509323,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0005859375,
|
|
"completions/max_length": 1318.4,
|
|
"completions/max_terminated_length": 633.8,
|
|
"completions/mean_length": 121.4595703125,
|
|
"completions/mean_terminated_length": 120.63011322021484,
|
|
"completions/min_length": 48.4,
|
|
"completions/min_terminated_length": 48.4,
|
|
"epoch": 0.144,
|
|
"grad_norm": 0.0008337603649124503,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 156501322.0,
|
|
"reward": 1.0360254526138306,
|
|
"reward_std": 0.11216316372156143,
|
|
"rewards/accuracy_reward": 0.3078125,
|
|
"rewards/brier_reward": 0.7661016464233399,
|
|
"rewards/confidence_one_or_zero": 0.10185546875,
|
|
"rewards/format_reward": 0.99814453125,
|
|
"rewards/mean_confidence_reward": 0.23704092800617219,
|
|
"rewards/smce": -0.08523178026080132,
|
|
"step": 45
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00068359375,
|
|
"completions/max_length": 1308.0,
|
|
"completions/max_terminated_length": 469.2,
|
|
"completions/mean_length": 122.78017578125,
|
|
"completions/mean_terminated_length": 121.81385955810546,
|
|
"completions/min_length": 54.0,
|
|
"completions/min_terminated_length": 54.0,
|
|
"epoch": 0.16,
|
|
"grad_norm": 0.0008199186413548887,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 172552831.0,
|
|
"reward": 1.0488179683685304,
|
|
"reward_std": 0.1010824665427208,
|
|
"rewards/accuracy_reward": 0.3365234375,
|
|
"rewards/brier_reward": 0.7621974587440491,
|
|
"rewards/confidence_one_or_zero": 0.06181640625,
|
|
"rewards/format_reward": 0.99892578125,
|
|
"rewards/mean_confidence_reward": 0.22926148474216462,
|
|
"rewards/smce": -0.11216033399105071,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.16,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 401.0,
|
|
"eval_completions/max_terminated_length": 401.0,
|
|
"eval_completions/mean_length": 125.22658920288086,
|
|
"eval_completions/mean_terminated_length": 125.22658920288086,
|
|
"eval_completions/min_length": 63.0,
|
|
"eval_completions/min_terminated_length": 63.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 172552831.0,
|
|
"eval_reward": 1.032656490802765,
|
|
"eval_reward_std": 0.1260393038392067,
|
|
"eval_rewards/accuracy_reward": 0.271484375,
|
|
"eval_rewards/brier_reward": 0.7938330173492432,
|
|
"eval_rewards/confidence_one_or_zero": 0.041015625,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.23232988268136978,
|
|
"eval_rewards/smce": -0.04869849234819412,
|
|
"eval_runtime": 12.3556,
|
|
"eval_samples_per_second": 40.468,
|
|
"eval_steps_per_second": 0.162,
|
|
"step": 50
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0005859375,
|
|
"completions/max_length": 1308.6,
|
|
"completions/max_terminated_length": 431.8,
|
|
"completions/mean_length": 125.6203125,
|
|
"completions/mean_terminated_length": 124.79365539550781,
|
|
"completions/min_length": 56.8,
|
|
"completions/min_terminated_length": 56.8,
|
|
"epoch": 0.176,
|
|
"grad_norm": 0.0008834801847115159,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 188904367.0,
|
|
"reward": 1.0448211193084718,
|
|
"reward_std": 0.09679851233959198,
|
|
"rewards/accuracy_reward": 0.31142578125,
|
|
"rewards/brier_reward": 0.779005253314972,
|
|
"rewards/confidence_one_or_zero": 0.03623046875,
|
|
"rewards/format_reward": 0.99921875,
|
|
"rewards/mean_confidence_reward": 0.24507814943790435,
|
|
"rewards/smce": -0.08033295050263405,
|
|
"step": 55
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.000390625,
|
|
"completions/max_length": 1106.6,
|
|
"completions/max_terminated_length": 387.2,
|
|
"completions/mean_length": 124.562109375,
|
|
"completions/mean_terminated_length": 124.01052703857422,
|
|
"completions/min_length": 56.4,
|
|
"completions/min_terminated_length": 56.4,
|
|
"epoch": 0.192,
|
|
"grad_norm": 0.0007722419104538858,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 205241867.0,
|
|
"reward": 1.050026798248291,
|
|
"reward_std": 0.09879964888095856,
|
|
"rewards/accuracy_reward": 0.3302734375,
|
|
"rewards/brier_reward": 0.77076256275177,
|
|
"rewards/confidence_one_or_zero": 0.0201171875,
|
|
"rewards/format_reward": 0.9990234375,
|
|
"rewards/mean_confidence_reward": 0.2705323278903961,
|
|
"rewards/smce": -0.06530658006668091,
|
|
"step": 60
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 822.2,
|
|
"completions/max_terminated_length": 369.6,
|
|
"completions/mean_length": 125.0603515625,
|
|
"completions/mean_terminated_length": 124.64741363525391,
|
|
"completions/min_length": 54.6,
|
|
"completions/min_terminated_length": 54.6,
|
|
"epoch": 0.208,
|
|
"grad_norm": 0.0008552237995900214,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 221543093.0,
|
|
"reward": 1.060682201385498,
|
|
"reward_std": 0.10427201688289642,
|
|
"rewards/accuracy_reward": 0.34794921875,
|
|
"rewards/brier_reward": 0.7738091349601746,
|
|
"rewards/confidence_one_or_zero": 0.00810546875,
|
|
"rewards/format_reward": 0.999609375,
|
|
"rewards/mean_confidence_reward": 0.3167231798171997,
|
|
"rewards/smce": -0.038938027992844584,
|
|
"step": 65
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 366.4,
|
|
"completions/max_terminated_length": 366.4,
|
|
"completions/mean_length": 123.75830078125,
|
|
"completions/mean_terminated_length": 123.75830078125,
|
|
"completions/min_length": 56.2,
|
|
"completions/min_terminated_length": 56.2,
|
|
"epoch": 0.224,
|
|
"grad_norm": 0.0009770386386662722,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 237879594.0,
|
|
"reward": 1.0563500642776489,
|
|
"reward_std": 0.10786637514829636,
|
|
"rewards/accuracy_reward": 0.32978515625,
|
|
"rewards/brier_reward": 0.7832106351852417,
|
|
"rewards/confidence_one_or_zero": 0.00361328125,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.34383692145347594,
|
|
"rewards/smce": -0.033423048071563245,
|
|
"step": 70
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 550.0,
|
|
"completions/max_terminated_length": 336.2,
|
|
"completions/mean_length": 123.43134765625,
|
|
"completions/mean_terminated_length": 123.29346313476563,
|
|
"completions/min_length": 58.4,
|
|
"completions/min_terminated_length": 58.4,
|
|
"epoch": 0.24,
|
|
"grad_norm": 0.0008802193333394825,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 254230027.0,
|
|
"reward": 1.061513566970825,
|
|
"reward_std": 0.10092030316591263,
|
|
"rewards/accuracy_reward": 0.3501953125,
|
|
"rewards/brier_reward": 0.7731273651123047,
|
|
"rewards/confidence_one_or_zero": 0.0021484375,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.34638115763664246,
|
|
"rewards/smce": -0.03192154616117478,
|
|
"step": 75
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 596.6,
|
|
"completions/max_terminated_length": 356.6,
|
|
"completions/mean_length": 123.50390625,
|
|
"completions/mean_terminated_length": 123.36609649658203,
|
|
"completions/min_length": 58.4,
|
|
"completions/min_terminated_length": 58.4,
|
|
"epoch": 0.256,
|
|
"grad_norm": 0.0008312573190778494,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 270830611.0,
|
|
"reward": 1.0700663805007935,
|
|
"reward_std": 0.10508463829755783,
|
|
"rewards/accuracy_reward": 0.35947265625,
|
|
"rewards/brier_reward": 0.7807585239410401,
|
|
"rewards/confidence_one_or_zero": 0.00205078125,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.35435292720794676,
|
|
"rewards/smce": -0.013978258706629276,
|
|
"step": 80
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 1058.2,
|
|
"completions/max_terminated_length": 351.2,
|
|
"completions/mean_length": 123.29365234375,
|
|
"completions/mean_terminated_length": 122.8795181274414,
|
|
"completions/min_length": 61.8,
|
|
"completions/min_terminated_length": 61.8,
|
|
"epoch": 0.272,
|
|
"grad_norm": 0.0009743980481289327,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 287208562.0,
|
|
"reward": 1.078542709350586,
|
|
"reward_std": 0.11005425304174424,
|
|
"rewards/accuracy_reward": 0.39365234375,
|
|
"rewards/brier_reward": 0.7639232754707337,
|
|
"rewards/confidence_one_or_zero": 0.00087890625,
|
|
"rewards/format_reward": 0.99951171875,
|
|
"rewards/mean_confidence_reward": 0.3724678099155426,
|
|
"rewards/smce": -0.026533697731792927,
|
|
"step": 85
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 372.4,
|
|
"completions/max_terminated_length": 372.4,
|
|
"completions/mean_length": 122.5517578125,
|
|
"completions/mean_terminated_length": 122.5517578125,
|
|
"completions/min_length": 57.4,
|
|
"completions/min_terminated_length": 57.4,
|
|
"epoch": 0.288,
|
|
"grad_norm": 0.0009167360258288682,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 303624452.0,
|
|
"reward": 1.0746608257293702,
|
|
"reward_std": 0.10677139461040497,
|
|
"rewards/accuracy_reward": 0.37080078125,
|
|
"rewards/brier_reward": 0.7785232663154602,
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.38262552618980405,
|
|
"rewards/smce": -0.031942544505000114,
|
|
"step": 90
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 326.6,
|
|
"completions/max_terminated_length": 326.6,
|
|
"completions/mean_length": 121.0849609375,
|
|
"completions/mean_terminated_length": 121.0849609375,
|
|
"completions/min_length": 59.6,
|
|
"completions/min_terminated_length": 59.6,
|
|
"epoch": 0.304,
|
|
"grad_norm": 0.0009152097627520561,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 319829130.0,
|
|
"reward": 1.0819915771484374,
|
|
"reward_std": 0.10794731378555297,
|
|
"rewards/accuracy_reward": 0.40126953125,
|
|
"rewards/brier_reward": 0.7629124164581299,
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4008802831172943,
|
|
"rewards/smce": -0.04182044714689255,
|
|
"step": 95
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 280.6,
|
|
"completions/max_terminated_length": 280.6,
|
|
"completions/mean_length": 117.4109375,
|
|
"completions/mean_terminated_length": 117.4109375,
|
|
"completions/min_length": 60.4,
|
|
"completions/min_terminated_length": 60.4,
|
|
"epoch": 0.32,
|
|
"grad_norm": 0.0009640919743105769,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 335988634.0,
|
|
"reward": 1.0821470260620116,
|
|
"reward_std": 0.1047831878066063,
|
|
"rewards/accuracy_reward": 0.39453125,
|
|
"rewards/brier_reward": 0.7697669267654419,
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.4091184735298157,
|
|
"rewards/smce": -0.049343763291835784,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.32,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 202.5,
|
|
"eval_completions/max_terminated_length": 202.5,
|
|
"eval_completions/mean_length": 114.07200622558594,
|
|
"eval_completions/mean_terminated_length": 114.07200622558594,
|
|
"eval_completions/min_length": 60.0,
|
|
"eval_completions/min_terminated_length": 60.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 335988634.0,
|
|
"eval_reward": 1.0488357543945312,
|
|
"eval_reward_std": 0.1998460590839386,
|
|
"eval_rewards/accuracy_reward": 0.314453125,
|
|
"eval_rewards/brier_reward": 0.783225953578949,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.3979492336511612,
|
|
"eval_rewards/smce": -0.0844060592353344,
|
|
"eval_runtime": 9.2582,
|
|
"eval_samples_per_second": 54.006,
|
|
"eval_steps_per_second": 0.216,
|
|
"step": 100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 314.0,
|
|
"completions/max_terminated_length": 314.0,
|
|
"completions/mean_length": 116.25380859375,
|
|
"completions/mean_terminated_length": 116.25380859375,
|
|
"completions/min_length": 59.0,
|
|
"completions/min_terminated_length": 59.0,
|
|
"epoch": 0.336,
|
|
"grad_norm": 0.0010559717193245888,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0004,
|
|
"num_tokens": 352170305.0,
|
|
"reward": 1.0908453702926635,
|
|
"reward_std": 0.10500690937042237,
|
|
"rewards/accuracy_reward": 0.40830078125,
|
|
"rewards/brier_reward": 0.7734902620315551,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.3929440498352051,
|
|
"rewards/smce": -0.03384199421852827,
|
|
"step": 105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 366.8,
|
|
"completions/max_terminated_length": 366.8,
|
|
"completions/mean_length": 121.5208984375,
|
|
"completions/mean_terminated_length": 121.5208984375,
|
|
"completions/min_length": 62.0,
|
|
"completions/min_terminated_length": 62.0,
|
|
"epoch": 0.352,
|
|
"grad_norm": 0.0010314425453543663,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0003,
|
|
"num_tokens": 368524887.0,
|
|
"reward": 1.0825308561325073,
|
|
"reward_std": 0.10590031743049622,
|
|
"rewards/accuracy_reward": 0.392578125,
|
|
"rewards/brier_reward": 0.7725841403007507,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.3955056965351105,
|
|
"rewards/smce": -0.036800354905426505,
|
|
"step": 110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 341.2,
|
|
"completions/max_terminated_length": 341.2,
|
|
"completions/mean_length": 133.803515625,
|
|
"completions/mean_terminated_length": 133.803515625,
|
|
"completions/min_length": 57.8,
|
|
"completions/min_terminated_length": 57.8,
|
|
"epoch": 0.368,
|
|
"grad_norm": 0.0009640640346333385,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 384988507.0,
|
|
"reward": 1.0816544771194458,
|
|
"reward_std": 0.10436138659715652,
|
|
"rewards/accuracy_reward": 0.3865234375,
|
|
"rewards/brier_reward": 0.7767881631851197,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.4014230787754059,
|
|
"rewards/smce": -0.034000173956155774,
|
|
"step": 115
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 702.4,
|
|
"completions/max_terminated_length": 454.8,
|
|
"completions/mean_length": 146.3341796875,
|
|
"completions/mean_terminated_length": 146.19812927246093,
|
|
"completions/min_length": 66.4,
|
|
"completions/min_terminated_length": 66.4,
|
|
"epoch": 0.384,
|
|
"grad_norm": 0.0008643779437988997,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 401478553.0,
|
|
"reward": 1.0923637866973877,
|
|
"reward_std": 0.10116445124149323,
|
|
"rewards/accuracy_reward": 0.4091796875,
|
|
"rewards/brier_reward": 0.7758468389511108,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.3995400667190552,
|
|
"rewards/smce": -0.06742004379630089,
|
|
"step": 120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 613.6,
|
|
"completions/max_terminated_length": 379.0,
|
|
"completions/mean_length": 149.774609375,
|
|
"completions/mean_terminated_length": 149.63934326171875,
|
|
"completions/min_length": 71.6,
|
|
"completions/min_terminated_length": 71.6,
|
|
"epoch": 0.4,
|
|
"grad_norm": 0.0008598100976087153,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 417872245.0,
|
|
"reward": 1.0924156427383422,
|
|
"reward_std": 0.11007419526576996,
|
|
"rewards/accuracy_reward": 0.4076171875,
|
|
"rewards/brier_reward": 0.77731374502182,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.404058700799942,
|
|
"rewards/smce": -0.028200492449104785,
|
|
"step": 125
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 665.2,
|
|
"completions/max_terminated_length": 443.2,
|
|
"completions/mean_length": 156.9810546875,
|
|
"completions/mean_terminated_length": 156.57687683105468,
|
|
"completions/min_length": 73.4,
|
|
"completions/min_terminated_length": 73.4,
|
|
"epoch": 0.416,
|
|
"grad_norm": 0.0010003651259467006,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 434623475.0,
|
|
"reward": 1.110920262336731,
|
|
"reward_std": 0.11143997013568878,
|
|
"rewards/accuracy_reward": 0.45322265625,
|
|
"rewards/brier_reward": 0.7691106915473938,
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
|
"rewards/format_reward": 0.99951171875,
|
|
"rewards/mean_confidence_reward": 0.41947434544563295,
|
|
"rewards/smce": -0.05290180444717407,
|
|
"step": 130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 393.6,
|
|
"completions/max_terminated_length": 393.6,
|
|
"completions/mean_length": 163.24091796875,
|
|
"completions/mean_terminated_length": 163.24091796875,
|
|
"completions/min_length": 72.2,
|
|
"completions/min_terminated_length": 72.2,
|
|
"epoch": 0.432,
|
|
"grad_norm": 0.0007602354162372649,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 451437494.0,
|
|
"reward": 1.0949999332427978,
|
|
"reward_std": 0.11146767735481262,
|
|
"rewards/accuracy_reward": 0.41318359375,
|
|
"rewards/brier_reward": 0.7768206119537353,
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.4301083981990814,
|
|
"rewards/smce": -0.052707664296031,
|
|
"step": 135
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 427.0,
|
|
"completions/max_terminated_length": 427.0,
|
|
"completions/mean_length": 165.24130859375,
|
|
"completions/mean_terminated_length": 165.24130859375,
|
|
"completions/min_length": 76.2,
|
|
"completions/min_terminated_length": 76.2,
|
|
"epoch": 0.448,
|
|
"grad_norm": 0.000916799355763942,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 468191741.0,
|
|
"reward": 1.1065703868865966,
|
|
"reward_std": 0.11395464688539506,
|
|
"rewards/accuracy_reward": 0.4322265625,
|
|
"rewards/brier_reward": 0.7810142278671265,
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.42705950140953064,
|
|
"rewards/smce": -0.032019305787980554,
|
|
"step": 140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 397.8,
|
|
"completions/max_terminated_length": 397.8,
|
|
"completions/mean_length": 169.6619140625,
|
|
"completions/mean_terminated_length": 169.6619140625,
|
|
"completions/min_length": 79.4,
|
|
"completions/min_terminated_length": 79.4,
|
|
"epoch": 0.464,
|
|
"grad_norm": 0.0007820753962732852,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 484876247.0,
|
|
"reward": 1.109752631187439,
|
|
"reward_std": 0.10953254997730255,
|
|
"rewards/accuracy_reward": 0.44111328125,
|
|
"rewards/brier_reward": 0.7784909725189209,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.4319294333457947,
|
|
"rewards/smce": -0.021954568102955818,
|
|
"step": 145
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 947.6,
|
|
"completions/max_terminated_length": 490.4,
|
|
"completions/mean_length": 171.62763671875,
|
|
"completions/mean_terminated_length": 171.36109313964843,
|
|
"completions/min_length": 81.2,
|
|
"completions/min_terminated_length": 81.2,
|
|
"epoch": 0.48,
|
|
"grad_norm": 0.0009613652364350855,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 501703954.0,
|
|
"reward": 1.1193360567092896,
|
|
"reward_std": 0.1150269329547882,
|
|
"rewards/accuracy_reward": 0.46259765625,
|
|
"rewards/brier_reward": 0.7764689922332764,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.999609375,
|
|
"rewards/mean_confidence_reward": 0.43756228089332583,
|
|
"rewards/smce": -0.047635878622531894,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.48,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 353.0,
|
|
"eval_completions/max_terminated_length": 353.0,
|
|
"eval_completions/mean_length": 173.3000717163086,
|
|
"eval_completions/mean_terminated_length": 173.3000717163086,
|
|
"eval_completions/min_length": 96.5,
|
|
"eval_completions/min_terminated_length": 96.5,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 501703954.0,
|
|
"eval_reward": 1.0731765031814575,
|
|
"eval_reward_std": 0.22983557730913162,
|
|
"eval_rewards/accuracy_reward": 0.36328125,
|
|
"eval_rewards/brier_reward": 0.7830783128738403,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.43638673424720764,
|
|
"eval_rewards/smce": -0.07412109896540642,
|
|
"eval_runtime": 12.0771,
|
|
"eval_samples_per_second": 41.401,
|
|
"eval_steps_per_second": 0.166,
|
|
"step": 150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 605.8,
|
|
"completions/max_terminated_length": 368.0,
|
|
"completions/mean_length": 173.3896484375,
|
|
"completions/mean_terminated_length": 173.25641174316405,
|
|
"completions/min_length": 87.0,
|
|
"completions/min_terminated_length": 87.0,
|
|
"epoch": 0.496,
|
|
"grad_norm": 0.0009212964214384556,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 518601576.0,
|
|
"reward": 1.118337345123291,
|
|
"reward_std": 0.110992231965065,
|
|
"rewards/accuracy_reward": 0.4560546875,
|
|
"rewards/brier_reward": 0.780720853805542,
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.44267138838768005,
|
|
"rewards/smce": -0.04033624045550823,
|
|
"step": 155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 418.8,
|
|
"completions/max_terminated_length": 418.8,
|
|
"completions/mean_length": 174.5041015625,
|
|
"completions/mean_terminated_length": 174.5041015625,
|
|
"completions/min_length": 86.6,
|
|
"completions/min_terminated_length": 86.6,
|
|
"epoch": 0.512,
|
|
"grad_norm": 0.0007764216279610991,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 535378450.0,
|
|
"reward": 1.1107932567596435,
|
|
"reward_std": 0.10566790401935577,
|
|
"rewards/accuracy_reward": 0.4388671875,
|
|
"rewards/brier_reward": 0.7827236652374268,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.43680567145347593,
|
|
"rewards/smce": -0.051772523671388626,
|
|
"step": 160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 634.8,
|
|
"completions/max_terminated_length": 437.2,
|
|
"completions/mean_length": 172.83876953125,
|
|
"completions/mean_terminated_length": 172.7058319091797,
|
|
"completions/min_length": 83.2,
|
|
"completions/min_terminated_length": 83.2,
|
|
"epoch": 0.528,
|
|
"grad_norm": 0.0008587085176259279,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 552035807.0,
|
|
"reward": 1.1129685878753661,
|
|
"reward_std": 0.10902059525251388,
|
|
"rewards/accuracy_reward": 0.4474609375,
|
|
"rewards/brier_reward": 0.7785768270492553,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.45129268169403075,
|
|
"rewards/smce": -0.038398644607514146,
|
|
"step": 165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 678.0,
|
|
"completions/max_terminated_length": 462.6,
|
|
"completions/mean_length": 171.39228515625,
|
|
"completions/mean_terminated_length": 171.1254852294922,
|
|
"completions/min_length": 84.0,
|
|
"completions/min_terminated_length": 84.0,
|
|
"epoch": 0.544,
|
|
"grad_norm": 0.0009459138382226229,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 568991088.0,
|
|
"reward": 1.0985527276992797,
|
|
"reward_std": 0.10920530259609222,
|
|
"rewards/accuracy_reward": 0.41337890625,
|
|
"rewards/brier_reward": 0.7839253544807434,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4351104199886322,
|
|
"rewards/smce": -0.04374492410570383,
|
|
"step": 170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 641.0,
|
|
"completions/max_terminated_length": 424.8,
|
|
"completions/mean_length": 174.48369140625,
|
|
"completions/mean_terminated_length": 174.3507507324219,
|
|
"completions/min_length": 84.4,
|
|
"completions/min_terminated_length": 84.4,
|
|
"epoch": 0.56,
|
|
"grad_norm": 0.0009580635232850909,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 585903977.0,
|
|
"reward": 1.1115816354751586,
|
|
"reward_std": 0.10534543991088867,
|
|
"rewards/accuracy_reward": 0.43564453125,
|
|
"rewards/brier_reward": 0.787619686126709,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.4416061580181122,
|
|
"rewards/smce": -0.04100565351545811,
|
|
"step": 175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 396.6,
|
|
"completions/max_terminated_length": 396.6,
|
|
"completions/mean_length": 173.34697265625,
|
|
"completions/mean_terminated_length": 173.34697265625,
|
|
"completions/min_length": 81.8,
|
|
"completions/min_terminated_length": 81.8,
|
|
"epoch": 0.576,
|
|
"grad_norm": 0.0008675847202539444,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 602717066.0,
|
|
"reward": 1.1025778532028199,
|
|
"reward_std": 0.09847408682107925,
|
|
"rewards/accuracy_reward": 0.4197265625,
|
|
"rewards/brier_reward": 0.7854331374168396,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.4456318259239197,
|
|
"rewards/smce": -0.04764346107840538,
|
|
"step": 180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 712.8,
|
|
"completions/max_terminated_length": 498.4,
|
|
"completions/mean_length": 173.9390625,
|
|
"completions/mean_terminated_length": 173.53940124511718,
|
|
"completions/min_length": 84.8,
|
|
"completions/min_terminated_length": 84.8,
|
|
"epoch": 0.592,
|
|
"grad_norm": 0.0009322913829237223,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 619666010.0,
|
|
"reward": 1.12094247341156,
|
|
"reward_std": 0.10624455362558365,
|
|
"rewards/accuracy_reward": 0.4673828125,
|
|
"rewards/brier_reward": 0.774797546863556,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.4424082159996033,
|
|
"rewards/smce": -0.03360339626669884,
|
|
"step": 185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 625.2,
|
|
"completions/max_terminated_length": 395.6,
|
|
"completions/mean_length": 176.16884765625,
|
|
"completions/mean_terminated_length": 176.03597717285157,
|
|
"completions/min_length": 85.4,
|
|
"completions/min_terminated_length": 85.4,
|
|
"epoch": 0.608,
|
|
"grad_norm": 0.0008757953182794154,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 636325915.0,
|
|
"reward": 1.1331445455551148,
|
|
"reward_std": 0.10666034519672393,
|
|
"rewards/accuracy_reward": 0.47880859375,
|
|
"rewards/brier_reward": 0.7875788450241089,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.47118945717811583,
|
|
"rewards/smce": -0.016280164942145348,
|
|
"step": 190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 463.2,
|
|
"completions/max_terminated_length": 463.2,
|
|
"completions/mean_length": 179.38330078125,
|
|
"completions/mean_terminated_length": 179.38330078125,
|
|
"completions/min_length": 84.8,
|
|
"completions/min_terminated_length": 84.8,
|
|
"epoch": 0.624,
|
|
"grad_norm": 0.0009400384151376784,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 653288912.0,
|
|
"reward": 1.1178892612457276,
|
|
"reward_std": 0.11412412077188491,
|
|
"rewards/accuracy_reward": 0.44765625,
|
|
"rewards/brier_reward": 0.7881268620491028,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.47951045632362366,
|
|
"rewards/smce": -0.05635717511177063,
|
|
"step": 195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 719.0,
|
|
"completions/max_terminated_length": 520.8,
|
|
"completions/mean_length": 180.0751953125,
|
|
"completions/mean_terminated_length": 179.94290771484376,
|
|
"completions/min_length": 85.6,
|
|
"completions/min_terminated_length": 85.6,
|
|
"epoch": 0.64,
|
|
"grad_norm": 0.0008365341345779598,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0002,
|
|
"num_tokens": 670308850.0,
|
|
"reward": 1.0954386472702027,
|
|
"reward_std": 0.11347412168979645,
|
|
"rewards/accuracy_reward": 0.40654296875,
|
|
"rewards/brier_reward": 0.7844377636909485,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.4732663094997406,
|
|
"rewards/smce": -0.06727753281593322,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.64,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 349.0,
|
|
"eval_completions/max_terminated_length": 349.0,
|
|
"eval_completions/mean_length": 184.80519104003906,
|
|
"eval_completions/mean_terminated_length": 184.80519104003906,
|
|
"eval_completions/min_length": 101.0,
|
|
"eval_completions/min_terminated_length": 101.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 670308850.0,
|
|
"eval_reward": 1.0808968544006348,
|
|
"eval_reward_std": 0.2522861063480377,
|
|
"eval_rewards/accuracy_reward": 0.36328125,
|
|
"eval_rewards/brier_reward": 0.7985226511955261,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.47593750059604645,
|
|
"eval_rewards/smce": -0.11298829317092896,
|
|
"eval_runtime": 12.1311,
|
|
"eval_samples_per_second": 41.217,
|
|
"eval_steps_per_second": 0.165,
|
|
"step": 200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 417.6,
|
|
"completions/max_terminated_length": 417.6,
|
|
"completions/mean_length": 180.13681640625,
|
|
"completions/mean_terminated_length": 180.13681640625,
|
|
"completions/min_length": 83.8,
|
|
"completions/min_terminated_length": 83.8,
|
|
"epoch": 0.656,
|
|
"grad_norm": 0.0008805691613815725,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 687058955.0,
|
|
"reward": 1.1146647214889527,
|
|
"reward_std": 0.10650343894958496,
|
|
"rewards/accuracy_reward": 0.4419921875,
|
|
"rewards/brier_reward": 0.7873411774635315,
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.48468555212020875,
|
|
"rewards/smce": -0.04835297502577305,
|
|
"step": 205
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 861.0,
|
|
"completions/max_terminated_length": 467.2,
|
|
"completions/mean_length": 182.21337890625,
|
|
"completions/mean_terminated_length": 181.81690979003906,
|
|
"completions/min_length": 89.6,
|
|
"completions/min_terminated_length": 89.6,
|
|
"epoch": 0.672,
|
|
"grad_norm": 0.0007954036700539291,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 704059892.0,
|
|
"reward": 1.1312153816223145,
|
|
"reward_std": 0.10522501021623612,
|
|
"rewards/accuracy_reward": 0.46982421875,
|
|
"rewards/brier_reward": 0.7929009675979615,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.46643847823143003,
|
|
"rewards/smce": -0.022859130799770356,
|
|
"step": 210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 416.8,
|
|
"completions/max_terminated_length": 416.8,
|
|
"completions/mean_length": 186.283203125,
|
|
"completions/mean_terminated_length": 186.283203125,
|
|
"completions/min_length": 91.2,
|
|
"completions/min_terminated_length": 91.2,
|
|
"epoch": 0.688,
|
|
"grad_norm": 0.000893253309186548,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 720923496.0,
|
|
"reward": 1.130001473426819,
|
|
"reward_std": 0.09947767555713653,
|
|
"rewards/accuracy_reward": 0.46083984375,
|
|
"rewards/brier_reward": 0.7991663455963135,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.4343766748905182,
|
|
"rewards/smce": -0.04040838629007339,
|
|
"step": 215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 746.2,
|
|
"completions/max_terminated_length": 589.8,
|
|
"completions/mean_length": 187.50205078125,
|
|
"completions/mean_terminated_length": 187.37029724121095,
|
|
"completions/min_length": 87.2,
|
|
"completions/min_terminated_length": 87.2,
|
|
"epoch": 0.704,
|
|
"grad_norm": 0.000925455882679671,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 737821181.0,
|
|
"reward": 1.1401437997817994,
|
|
"reward_std": 0.10884704440832138,
|
|
"rewards/accuracy_reward": 0.49033203125,
|
|
"rewards/brier_reward": 0.7900588274002075,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.47714354991912844,
|
|
"rewards/smce": -0.06580102406442165,
|
|
"step": 220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 895.4,
|
|
"completions/max_terminated_length": 455.0,
|
|
"completions/mean_length": 187.88642578125,
|
|
"completions/mean_terminated_length": 187.4909240722656,
|
|
"completions/min_length": 92.2,
|
|
"completions/min_terminated_length": 92.2,
|
|
"epoch": 0.72,
|
|
"grad_norm": 0.0009635393507778645,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 754833394.0,
|
|
"reward": 1.1586825370788574,
|
|
"reward_std": 0.10534119158983231,
|
|
"rewards/accuracy_reward": 0.51337890625,
|
|
"rewards/brier_reward": 0.8042813301086426,
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.48370898962020875,
|
|
"rewards/smce": -0.031232983246445654,
|
|
"step": 225
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 469.0,
|
|
"completions/max_terminated_length": 469.0,
|
|
"completions/mean_length": 186.92626953125,
|
|
"completions/mean_terminated_length": 186.92626953125,
|
|
"completions/min_length": 93.2,
|
|
"completions/min_terminated_length": 93.2,
|
|
"epoch": 0.736,
|
|
"grad_norm": 0.0009669368155300617,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 771878079.0,
|
|
"reward": 1.1391812324523927,
|
|
"reward_std": 0.11643845587968826,
|
|
"rewards/accuracy_reward": 0.48623046875,
|
|
"rewards/brier_reward": 0.7922345280647278,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.5351982235908508,
|
|
"rewards/smce": -0.05878893509507179,
|
|
"step": 230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 751.2,
|
|
"completions/max_terminated_length": 566.0,
|
|
"completions/mean_length": 183.4779296875,
|
|
"completions/mean_terminated_length": 183.34601135253905,
|
|
"completions/min_length": 85.8,
|
|
"completions/min_terminated_length": 85.8,
|
|
"epoch": 0.752,
|
|
"grad_norm": 0.0010671571362763643,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 788732317.0,
|
|
"reward": 1.1412156581878663,
|
|
"reward_std": 0.12025659680366516,
|
|
"rewards/accuracy_reward": 0.4857421875,
|
|
"rewards/brier_reward": 0.7967909932136535,
|
|
"rewards/confidence_one_or_zero": 0.00087890625,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.5363173723220825,
|
|
"rewards/smce": -0.0527303846552968,
|
|
"step": 235
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 710.0,
|
|
"completions/max_terminated_length": 519.6,
|
|
"completions/mean_length": 183.93583984375,
|
|
"completions/mean_terminated_length": 183.8036376953125,
|
|
"completions/min_length": 88.4,
|
|
"completions/min_terminated_length": 88.4,
|
|
"epoch": 0.768,
|
|
"grad_norm": 0.0010159483645111322,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 805502060.0,
|
|
"reward": 1.1255807876586914,
|
|
"reward_std": 0.11585851460695266,
|
|
"rewards/accuracy_reward": 0.4578125,
|
|
"rewards/brier_reward": 0.7936447501182556,
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.4842294991016388,
|
|
"rewards/smce": -0.03694276623427868,
|
|
"step": 240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 686.8,
|
|
"completions/max_terminated_length": 517.8,
|
|
"completions/mean_length": 184.45654296875,
|
|
"completions/mean_terminated_length": 184.32433166503907,
|
|
"completions/min_length": 90.4,
|
|
"completions/min_terminated_length": 90.4,
|
|
"epoch": 0.784,
|
|
"grad_norm": 0.0009062938624992967,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 822614735.0,
|
|
"reward": 1.145763921737671,
|
|
"reward_std": 0.1053706169128418,
|
|
"rewards/accuracy_reward": 0.503125,
|
|
"rewards/brier_reward": 0.7886038422584534,
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4539990246295929,
|
|
"rewards/smce": -0.06559372935444116,
|
|
"step": 245
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 458.4,
|
|
"completions/max_terminated_length": 458.4,
|
|
"completions/mean_length": 182.2365234375,
|
|
"completions/mean_terminated_length": 182.2365234375,
|
|
"completions/min_length": 89.6,
|
|
"completions/min_terminated_length": 89.6,
|
|
"epoch": 0.8,
|
|
"grad_norm": 0.0009169210097752512,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 839356229.0,
|
|
"reward": 1.1485914945602418,
|
|
"reward_std": 0.11149813830852509,
|
|
"rewards/accuracy_reward": 0.4984375,
|
|
"rewards/brier_reward": 0.7987475156784057,
|
|
"rewards/confidence_one_or_zero": 0.0015625,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.5062128841876984,
|
|
"rewards/smce": -0.030764539353549482,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.8,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 378.0,
|
|
"eval_completions/max_terminated_length": 378.0,
|
|
"eval_completions/mean_length": 186.12918853759766,
|
|
"eval_completions/mean_terminated_length": 186.12918853759766,
|
|
"eval_completions/min_length": 100.0,
|
|
"eval_completions/min_terminated_length": 100.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 839356229.0,
|
|
"eval_reward": 1.097034513950348,
|
|
"eval_reward_std": 0.2692975252866745,
|
|
"eval_rewards/accuracy_reward": 0.396484375,
|
|
"eval_rewards/brier_reward": 0.7975927591323853,
|
|
"eval_rewards/confidence_one_or_zero": 0.001953125,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.4864257723093033,
|
|
"eval_rewards/smce": -0.09077637270092964,
|
|
"eval_runtime": 12.4371,
|
|
"eval_samples_per_second": 40.202,
|
|
"eval_steps_per_second": 0.161,
|
|
"step": 250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 489.8,
|
|
"completions/max_terminated_length": 489.8,
|
|
"completions/mean_length": 186.72607421875,
|
|
"completions/mean_terminated_length": 186.72607421875,
|
|
"completions/min_length": 90.0,
|
|
"completions/min_terminated_length": 90.0,
|
|
"epoch": 0.816,
|
|
"grad_norm": 0.0009517151629552245,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 856412304.0,
|
|
"reward": 1.1400040626525878,
|
|
"reward_std": 0.10449277609586716,
|
|
"rewards/accuracy_reward": 0.48359375,
|
|
"rewards/brier_reward": 0.796417486667633,
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.4969150543212891,
|
|
"rewards/smce": -0.041142824292182925,
|
|
"step": 255
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 907.2,
|
|
"completions/max_terminated_length": 500.8,
|
|
"completions/mean_length": 193.9853515625,
|
|
"completions/mean_terminated_length": 193.72337951660157,
|
|
"completions/min_length": 95.8,
|
|
"completions/min_terminated_length": 95.8,
|
|
"epoch": 0.832,
|
|
"grad_norm": 0.0006846592877991498,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 873349946.0,
|
|
"reward": 1.1227067232131958,
|
|
"reward_std": 0.10306735038757324,
|
|
"rewards/accuracy_reward": 0.44794921875,
|
|
"rewards/brier_reward": 0.7976632714271545,
|
|
"rewards/confidence_one_or_zero": 0.001171875,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4825107455253601,
|
|
"rewards/smce": -0.04679301045835018,
|
|
"step": 260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 886.4,
|
|
"completions/max_terminated_length": 466.2,
|
|
"completions/mean_length": 199.3931640625,
|
|
"completions/mean_terminated_length": 199.1320373535156,
|
|
"completions/min_length": 92.2,
|
|
"completions/min_terminated_length": 92.2,
|
|
"epoch": 0.848,
|
|
"grad_norm": 0.0008242399781011045,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 890447252.0,
|
|
"reward": 1.1360819339752197,
|
|
"reward_std": 0.08981756120920181,
|
|
"rewards/accuracy_reward": 0.4703125,
|
|
"rewards/brier_reward": 0.8020532727241516,
|
|
"rewards/confidence_one_or_zero": 0.00244140625,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.41491699814796446,
|
|
"rewards/smce": -0.07371992468833924,
|
|
"step": 265
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 720.0,
|
|
"completions/max_terminated_length": 512.4,
|
|
"completions/mean_length": 201.26787109375,
|
|
"completions/mean_terminated_length": 201.13762817382812,
|
|
"completions/min_length": 99.0,
|
|
"completions/min_terminated_length": 99.0,
|
|
"epoch": 0.864,
|
|
"grad_norm": 0.0007919904310256243,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 907681771.0,
|
|
"reward": 1.1619311332702638,
|
|
"reward_std": 0.08479854315519333,
|
|
"rewards/accuracy_reward": 0.52314453125,
|
|
"rewards/brier_reward": 0.8008243322372437,
|
|
"rewards/confidence_one_or_zero": 0.001171875,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.4348466873168945,
|
|
"rewards/smce": -0.097522396966815,
|
|
"step": 270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 1168.4,
|
|
"completions/max_terminated_length": 544.4,
|
|
"completions/mean_length": 203.99453125,
|
|
"completions/mean_terminated_length": 203.60350646972657,
|
|
"completions/min_length": 96.4,
|
|
"completions/min_terminated_length": 96.4,
|
|
"epoch": 0.88,
|
|
"grad_norm": 0.0009532640688121319,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 924848563.0,
|
|
"reward": 1.16089608669281,
|
|
"reward_std": 0.10618025660514832,
|
|
"rewards/accuracy_reward": 0.52587890625,
|
|
"rewards/brier_reward": 0.7962094664573669,
|
|
"rewards/confidence_one_or_zero": 0.00107421875,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.5438671946525574,
|
|
"rewards/smce": -0.04220922961831093,
|
|
"step": 275
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 708.0,
|
|
"completions/max_terminated_length": 502.2,
|
|
"completions/mean_length": 209.7365234375,
|
|
"completions/mean_terminated_length": 209.60684814453126,
|
|
"completions/min_length": 103.6,
|
|
"completions/min_terminated_length": 103.6,
|
|
"epoch": 0.896,
|
|
"grad_norm": 0.0010002513881772757,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 941986857.0,
|
|
"reward": 1.1198976755142211,
|
|
"reward_std": 0.11406386941671372,
|
|
"rewards/accuracy_reward": 0.4515625,
|
|
"rewards/brier_reward": 0.7884399652481079,
|
|
"rewards/confidence_one_or_zero": 0.00078125,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.5795322299003601,
|
|
"rewards/smce": -0.12874267250299454,
|
|
"step": 280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 551.0,
|
|
"completions/max_terminated_length": 551.0,
|
|
"completions/mean_length": 207.69072265625,
|
|
"completions/mean_terminated_length": 207.69072265625,
|
|
"completions/min_length": 105.2,
|
|
"completions/min_terminated_length": 105.2,
|
|
"epoch": 0.912,
|
|
"grad_norm": 0.0007915336755104363,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 959217642.0,
|
|
"reward": 1.1549952268600463,
|
|
"reward_std": 0.10957214832305909,
|
|
"rewards/accuracy_reward": 0.5193359375,
|
|
"rewards/brier_reward": 0.7906574726104736,
|
|
"rewards/confidence_one_or_zero": 0.00146484375,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.5557373106479645,
|
|
"rewards/smce": -0.0403962416574359,
|
|
"step": 285
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 558.0,
|
|
"completions/max_terminated_length": 558.0,
|
|
"completions/mean_length": 215.7451171875,
|
|
"completions/mean_terminated_length": 215.7451171875,
|
|
"completions/min_length": 113.4,
|
|
"completions/min_terminated_length": 113.4,
|
|
"epoch": 0.928,
|
|
"grad_norm": 0.0009195299935527146,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 976234712.0,
|
|
"reward": 1.1424710273742675,
|
|
"reward_std": 0.10220244973897934,
|
|
"rewards/accuracy_reward": 0.47255859375,
|
|
"rewards/brier_reward": 0.8123879432678223,
|
|
"rewards/confidence_one_or_zero": 0.00302734375,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.4893798828125,
|
|
"rewards/smce": -0.05421997383236885,
|
|
"step": 290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 1016.4,
|
|
"completions/max_terminated_length": 618.6,
|
|
"completions/mean_length": 226.06904296875,
|
|
"completions/mean_terminated_length": 225.81295776367188,
|
|
"completions/min_length": 113.4,
|
|
"completions/min_terminated_length": 113.4,
|
|
"epoch": 0.944,
|
|
"grad_norm": 0.0008430502493865788,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 993718171.0,
|
|
"reward": 1.1584136962890625,
|
|
"reward_std": 0.08734249025583267,
|
|
"rewards/accuracy_reward": 0.50771484375,
|
|
"rewards/brier_reward": 0.8093124866485596,
|
|
"rewards/confidence_one_or_zero": 0.00087890625,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.45402538776397705,
|
|
"rewards/smce": -0.05462546311318874,
|
|
"step": 295
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 946.4,
|
|
"completions/max_terminated_length": 563.4,
|
|
"completions/mean_length": 231.7501953125,
|
|
"completions/mean_terminated_length": 231.4956481933594,
|
|
"completions/min_length": 109.2,
|
|
"completions/min_terminated_length": 109.2,
|
|
"epoch": 0.96,
|
|
"grad_norm": 0.0007768862997181714,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 1011168445.0,
|
|
"reward": 1.1429346323013305,
|
|
"reward_std": 0.10092819035053253,
|
|
"rewards/accuracy_reward": 0.48017578125,
|
|
"rewards/brier_reward": 0.8058898329734803,
|
|
"rewards/confidence_one_or_zero": 0.00302734375,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4817968726158142,
|
|
"rewards/smce": -0.019481103867292404,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.96,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 460.5,
|
|
"eval_completions/max_terminated_length": 460.5,
|
|
"eval_completions/mean_length": 238.47315979003906,
|
|
"eval_completions/mean_terminated_length": 238.47315979003906,
|
|
"eval_completions/min_length": 115.0,
|
|
"eval_completions/min_terminated_length": 115.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 1011168445.0,
|
|
"eval_reward": 1.0985642075538635,
|
|
"eval_reward_std": 0.27343614399433136,
|
|
"eval_rewards/accuracy_reward": 0.38671875,
|
|
"eval_rewards/brier_reward": 0.8104199469089508,
|
|
"eval_rewards/confidence_one_or_zero": 0.001953125,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.4990234375,
|
|
"eval_rewards/smce": -0.11257324367761612,
|
|
"eval_runtime": 14.2596,
|
|
"eval_samples_per_second": 35.064,
|
|
"eval_steps_per_second": 0.14,
|
|
"step": 300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 1123.8,
|
|
"completions/max_terminated_length": 628.0,
|
|
"completions/mean_length": 237.015234375,
|
|
"completions/mean_terminated_length": 236.63544921875,
|
|
"completions/min_length": 118.8,
|
|
"completions/min_terminated_length": 118.8,
|
|
"epoch": 0.976,
|
|
"grad_norm": 0.0008644487243145704,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 1028736857.0,
|
|
"reward": 1.1499141216278077,
|
|
"reward_std": 0.0988913893699646,
|
|
"rewards/accuracy_reward": 0.47451171875,
|
|
"rewards/brier_reward": 0.8256130218505859,
|
|
"rewards/confidence_one_or_zero": 0.00322265625,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.5035009860992432,
|
|
"rewards/smce": -0.044948733597993853,
|
|
"step": 305
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 892.0,
|
|
"completions/max_terminated_length": 709.0,
|
|
"completions/mean_length": 235.7791015625,
|
|
"completions/mean_terminated_length": 235.65211791992186,
|
|
"completions/min_length": 118.2,
|
|
"completions/min_terminated_length": 118.2,
|
|
"epoch": 0.992,
|
|
"grad_norm": 0.000783053576014936,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 1046055459.0,
|
|
"reward": 1.1583145380020141,
|
|
"reward_std": 0.09551425129175187,
|
|
"rewards/accuracy_reward": 0.52236328125,
|
|
"rewards/brier_reward": 0.794464111328125,
|
|
"rewards/confidence_one_or_zero": 0.00166015625,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.5242343783378601,
|
|
"rewards/smce": -0.040295251086354254,
|
|
"step": 310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00048828125,
|
|
"completions/max_length": 1042.5,
|
|
"completions/max_terminated_length": 762.5,
|
|
"completions/mean_length": 237.88067626953125,
|
|
"completions/mean_terminated_length": 237.24916076660156,
|
|
"completions/min_length": 113.0,
|
|
"completions/min_terminated_length": 113.0,
|
|
"epoch": 0.9984,
|
|
"num_tokens": 1053120646.0,
|
|
"reward": 1.1611324548721313,
|
|
"reward_std": 0.10324221849441528,
|
|
"rewards/accuracy_reward": 0.515380859375,
|
|
"rewards/brier_reward": 0.807620108127594,
|
|
"rewards/confidence_one_or_zero": 0.0009765625,
|
|
"rewards/format_reward": 0.999267578125,
|
|
"rewards/mean_confidence_reward": 0.4963721036911011,
|
|
"rewards/smce": -0.04539550840854645,
|
|
"step": 312,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.00408483054268753,
|
|
"train_runtime": 42881.0261,
|
|
"train_samples_per_second": 0.466,
|
|
"train_steps_per_second": 0.007
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 312,
|
|
"num_input_tokens_seen": 1053120646,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 60,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|