1944 lines
72 KiB
JSON
1944 lines
72 KiB
JSON
{
|
|
"best_metric": null,
|
|
"best_model_checkpoint": null,
|
|
"epoch": 0.9984,
|
|
"eval_steps": 50,
|
|
"global_step": 312,
|
|
"is_hyper_param_search": false,
|
|
"is_local_process_zero": true,
|
|
"is_world_process_zero": true,
|
|
"log_history": [
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.1080078125,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1530.4,
|
|
"completions/mean_length": 468.44228515625,
|
|
"completions/mean_terminated_length": 339.2684265136719,
|
|
"completions/min_length": 1.0,
|
|
"completions/min_terminated_length": 1.0,
|
|
"epoch": 0.016,
|
|
"grad_norm": 0.003270028391852975,
|
|
"learning_rate": 3.1249999999999997e-07,
|
|
"loss": 0.0516,
|
|
"num_tokens": 19640881.0,
|
|
"reward": 0.22072359919548035,
|
|
"reward_std": 0.3922470033168793,
|
|
"rewards/accuracy_reward": 0.065625,
|
|
"rewards/brier_reward": 0.11248895078897476,
|
|
"rewards/confidence_one_or_zero": 0.4205078125,
|
|
"rewards/format_reward": 0.26337890625,
|
|
"rewards/mean_confidence_reward": 0.540982985496521,
|
|
"rewards/smce": -0.4757453978061676,
|
|
"step": 5
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.10869140625,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1529.0,
|
|
"completions/mean_length": 454.3373046875,
|
|
"completions/mean_terminated_length": 322.4891845703125,
|
|
"completions/min_length": 1.0,
|
|
"completions/min_terminated_length": 1.0,
|
|
"epoch": 0.032,
|
|
"grad_norm": 0.004311034921556711,
|
|
"learning_rate": 6.249999999999999e-07,
|
|
"loss": 0.0536,
|
|
"num_tokens": 39393647.0,
|
|
"reward": 0.2296316921710968,
|
|
"reward_std": 0.38993753790855407,
|
|
"rewards/accuracy_reward": 0.0630859375,
|
|
"rewards/brier_reward": 0.11341342478990554,
|
|
"rewards/confidence_one_or_zero": 0.4353515625,
|
|
"rewards/format_reward": 0.2828125,
|
|
"rewards/mean_confidence_reward": 0.5675621032714844,
|
|
"rewards/smce": -0.5047040998935699,
|
|
"step": 10
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.05458984375,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1483.2,
|
|
"completions/mean_length": 319.65009765625,
|
|
"completions/mean_terminated_length": 250.00528259277343,
|
|
"completions/min_length": 1.4,
|
|
"completions/min_terminated_length": 1.4,
|
|
"epoch": 0.048,
|
|
"grad_norm": 0.004322660155594349,
|
|
"learning_rate": 9.374999999999999e-07,
|
|
"loss": 0.0551,
|
|
"num_tokens": 57715600.0,
|
|
"reward": 0.40282471776008605,
|
|
"reward_std": 0.4364392518997192,
|
|
"rewards/accuracy_reward": 0.10986328125,
|
|
"rewards/brier_reward": 0.19759979844093323,
|
|
"rewards/confidence_one_or_zero": 0.5287109375,
|
|
"rewards/format_reward": 0.4982421875,
|
|
"rewards/mean_confidence_reward": 0.6919785380363465,
|
|
"rewards/smce": -0.582571291923523,
|
|
"step": 15
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.022265625,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1380.8,
|
|
"completions/mean_length": 212.587109375,
|
|
"completions/mean_terminated_length": 182.60447998046874,
|
|
"completions/min_length": 10.6,
|
|
"completions/min_terminated_length": 10.6,
|
|
"epoch": 0.064,
|
|
"grad_norm": 0.0025710673071444035,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0366,
|
|
"num_tokens": 74810892.0,
|
|
"reward": 0.6773836731910705,
|
|
"reward_std": 0.3933659136295319,
|
|
"rewards/accuracy_reward": 0.18837890625,
|
|
"rewards/brier_reward": 0.3333409786224365,
|
|
"rewards/confidence_one_or_zero": 0.5728515625,
|
|
"rewards/format_reward": 0.83310546875,
|
|
"rewards/mean_confidence_reward": 0.7932586193084716,
|
|
"rewards/smce": -0.6074068307876587,
|
|
"step": 20
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.011328125,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1245.4,
|
|
"completions/mean_length": 162.02568359375,
|
|
"completions/mean_terminated_length": 146.2960693359375,
|
|
"completions/min_length": 30.2,
|
|
"completions/min_terminated_length": 30.2,
|
|
"epoch": 0.08,
|
|
"grad_norm": 0.002596383448690176,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0178,
|
|
"num_tokens": 91403187.0,
|
|
"reward": 0.7920157313346863,
|
|
"reward_std": 0.32526588439941406,
|
|
"rewards/accuracy_reward": 0.2154296875,
|
|
"rewards/brier_reward": 0.40586367845535276,
|
|
"rewards/confidence_one_or_zero": 0.5203125,
|
|
"rewards/format_reward": 0.96279296875,
|
|
"rewards/mean_confidence_reward": 0.7853312015533447,
|
|
"rewards/smce": -0.5747557759284974,
|
|
"step": 25
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00869140625,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 1347.0,
|
|
"completions/mean_length": 149.4373046875,
|
|
"completions/mean_terminated_length": 137.28244323730468,
|
|
"completions/min_length": 28.8,
|
|
"completions/min_terminated_length": 28.8,
|
|
"epoch": 0.096,
|
|
"grad_norm": 0.0020388758275657892,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0135,
|
|
"num_tokens": 107978033.0,
|
|
"reward": 0.8774016618728637,
|
|
"reward_std": 0.2836435556411743,
|
|
"rewards/accuracy_reward": 0.23779296875,
|
|
"rewards/brier_reward": 0.5336522459983826,
|
|
"rewards/confidence_one_or_zero": 0.34033203125,
|
|
"rewards/format_reward": 0.9833984375,
|
|
"rewards/mean_confidence_reward": 0.6492803931236267,
|
|
"rewards/smce": -0.4229824423789978,
|
|
"step": 30
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0033203125,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 892.0,
|
|
"completions/mean_length": 135.47197265625,
|
|
"completions/mean_terminated_length": 130.8082702636719,
|
|
"completions/min_length": 41.8,
|
|
"completions/min_terminated_length": 41.8,
|
|
"epoch": 0.112,
|
|
"grad_norm": 0.001400059787556529,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0061,
|
|
"num_tokens": 124474738.0,
|
|
"reward": 0.9876136898994445,
|
|
"reward_std": 0.21337922811508178,
|
|
"rewards/accuracy_reward": 0.280859375,
|
|
"rewards/brier_reward": 0.7013182640075684,
|
|
"rewards/confidence_one_or_zero": 0.19404296875,
|
|
"rewards/format_reward": 0.99306640625,
|
|
"rewards/mean_confidence_reward": 0.4260053992271423,
|
|
"rewards/smce": -0.17858310043811798,
|
|
"step": 35
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00185546875,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 760.8,
|
|
"completions/mean_length": 130.4890625,
|
|
"completions/mean_terminated_length": 127.876953125,
|
|
"completions/min_length": 42.8,
|
|
"completions/min_terminated_length": 42.8,
|
|
"epoch": 0.128,
|
|
"grad_norm": 0.0009865883039310575,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.004,
|
|
"num_tokens": 140727618.0,
|
|
"reward": 1.0204854726791381,
|
|
"reward_std": 0.14821541607379912,
|
|
"rewards/accuracy_reward": 0.2814453125,
|
|
"rewards/brier_reward": 0.7634376049041748,
|
|
"rewards/confidence_one_or_zero": 0.13564453125,
|
|
"rewards/format_reward": 0.99609375,
|
|
"rewards/mean_confidence_reward": 0.3013033330440521,
|
|
"rewards/smce": -0.06513335853815079,
|
|
"step": 40
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00068359375,
|
|
"completions/max_length": 1376.6,
|
|
"completions/max_terminated_length": 551.0,
|
|
"completions/mean_length": 126.53388671875,
|
|
"completions/mean_terminated_length": 125.56988830566407,
|
|
"completions/min_length": 45.6,
|
|
"completions/min_terminated_length": 45.6,
|
|
"epoch": 0.144,
|
|
"grad_norm": 0.0008671495015732944,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 156973757.0,
|
|
"reward": 1.0439672470092773,
|
|
"reward_std": 0.12052493244409561,
|
|
"rewards/accuracy_reward": 0.32958984375,
|
|
"rewards/brier_reward": 0.7605997204780579,
|
|
"rewards/confidence_one_or_zero": 0.10439453125,
|
|
"rewards/format_reward": 0.99775390625,
|
|
"rewards/mean_confidence_reward": 0.2419735610485077,
|
|
"rewards/smce": -0.09777832627296448,
|
|
"step": 45
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00068359375,
|
|
"completions/max_length": 1223.6,
|
|
"completions/max_terminated_length": 700.8,
|
|
"completions/mean_length": 128.12744140625,
|
|
"completions/mean_terminated_length": 127.1651123046875,
|
|
"completions/min_length": 45.8,
|
|
"completions/min_terminated_length": 45.8,
|
|
"epoch": 0.16,
|
|
"grad_norm": 0.0007608377491123974,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 173306710.0,
|
|
"reward": 1.0441123008728028,
|
|
"reward_std": 0.10170394033193589,
|
|
"rewards/accuracy_reward": 0.322265625,
|
|
"rewards/brier_reward": 0.7680190086364747,
|
|
"rewards/confidence_one_or_zero": 0.06083984375,
|
|
"rewards/format_reward": 0.99794921875,
|
|
"rewards/mean_confidence_reward": 0.22877179086208344,
|
|
"rewards/smce": -0.09797133281826972,
|
|
"step": 50
|
|
},
|
|
{
|
|
"epoch": 0.16,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 296.5,
|
|
"eval_completions/max_terminated_length": 296.5,
|
|
"eval_completions/mean_length": 129.6832733154297,
|
|
"eval_completions/mean_terminated_length": 129.6832733154297,
|
|
"eval_completions/min_length": 69.0,
|
|
"eval_completions/min_terminated_length": 69.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 173306710.0,
|
|
"eval_reward": 1.0249468684196472,
|
|
"eval_reward_std": 0.14161328971385956,
|
|
"eval_rewards/accuracy_reward": 0.251953125,
|
|
"eval_rewards/brier_reward": 0.8018492162227631,
|
|
"eval_rewards/confidence_one_or_zero": 0.041015625,
|
|
"eval_rewards/format_reward": 0.99609375,
|
|
"eval_rewards/mean_confidence_reward": 0.24041016399860382,
|
|
"eval_rewards/smce": -0.02927441243082285,
|
|
"eval_runtime": 10.8431,
|
|
"eval_samples_per_second": 46.112,
|
|
"eval_steps_per_second": 0.184,
|
|
"step": 50
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 1195.4,
|
|
"completions/max_terminated_length": 487.8,
|
|
"completions/mean_length": 129.540625,
|
|
"completions/mean_terminated_length": 129.12831268310546,
|
|
"completions/min_length": 55.6,
|
|
"completions/min_terminated_length": 55.6,
|
|
"epoch": 0.176,
|
|
"grad_norm": 0.0007807699730619788,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 189870326.0,
|
|
"reward": 1.0457116842269898,
|
|
"reward_std": 0.09513775557279587,
|
|
"rewards/accuracy_reward": 0.3138671875,
|
|
"rewards/brier_reward": 0.778539526462555,
|
|
"rewards/confidence_one_or_zero": 0.03935546875,
|
|
"rewards/format_reward": 0.9990234375,
|
|
"rewards/mean_confidence_reward": 0.2438169986009598,
|
|
"rewards/smce": -0.07360489293932915,
|
|
"step": 55
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00048828125,
|
|
"completions/max_length": 1074.6,
|
|
"completions/max_terminated_length": 433.8,
|
|
"completions/mean_length": 131.770703125,
|
|
"completions/mean_terminated_length": 131.08482055664064,
|
|
"completions/min_length": 59.8,
|
|
"completions/min_terminated_length": 59.8,
|
|
"epoch": 0.192,
|
|
"grad_norm": 0.0007883945945650339,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0012,
|
|
"num_tokens": 206034474.0,
|
|
"reward": 1.0633267641067505,
|
|
"reward_std": 0.1018825501203537,
|
|
"rewards/accuracy_reward": 0.3580078125,
|
|
"rewards/brier_reward": 0.7696295857429505,
|
|
"rewards/confidence_one_or_zero": 0.0146484375,
|
|
"rewards/format_reward": 0.9990234375,
|
|
"rewards/mean_confidence_reward": 0.28199089169502256,
|
|
"rewards/smce": -0.07945134602487088,
|
|
"step": 60
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 837.0,
|
|
"completions/max_terminated_length": 386.0,
|
|
"completions/mean_length": 135.3244140625,
|
|
"completions/mean_terminated_length": 135.05081176757812,
|
|
"completions/min_length": 59.8,
|
|
"completions/min_terminated_length": 59.8,
|
|
"epoch": 0.208,
|
|
"grad_norm": 0.0007650046609342098,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 222452420.0,
|
|
"reward": 1.0677101373672486,
|
|
"reward_std": 0.10990563184022903,
|
|
"rewards/accuracy_reward": 0.36865234375,
|
|
"rewards/brier_reward": 0.7672603011131287,
|
|
"rewards/confidence_one_or_zero": 0.00673828125,
|
|
"rewards/format_reward": 0.99951171875,
|
|
"rewards/mean_confidence_reward": 0.32444043159484864,
|
|
"rewards/smce": -0.0480029359459877,
|
|
"step": 65
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00048828125,
|
|
"completions/max_length": 1536.0,
|
|
"completions/max_terminated_length": 369.8,
|
|
"completions/mean_length": 136.843359375,
|
|
"completions/mean_terminated_length": 136.15984497070312,
|
|
"completions/min_length": 59.2,
|
|
"completions/min_terminated_length": 59.2,
|
|
"epoch": 0.224,
|
|
"grad_norm": 0.000912521849386394,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0018,
|
|
"num_tokens": 239006880.0,
|
|
"reward": 1.0661633729934692,
|
|
"reward_std": 0.112481489777565,
|
|
"rewards/accuracy_reward": 0.36142578125,
|
|
"rewards/brier_reward": 0.7715874671936035,
|
|
"rewards/confidence_one_or_zero": 0.00546875,
|
|
"rewards/format_reward": 0.99931640625,
|
|
"rewards/mean_confidence_reward": 0.3463192105293274,
|
|
"rewards/smce": -0.03602232336997986,
|
|
"step": 70
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 693.8,
|
|
"completions/max_terminated_length": 523.8,
|
|
"completions/mean_length": 140.01494140625,
|
|
"completions/mean_terminated_length": 139.87867736816406,
|
|
"completions/min_length": 57.2,
|
|
"completions/min_terminated_length": 57.2,
|
|
"epoch": 0.24,
|
|
"grad_norm": 0.0008707091328687966,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 255692313.0,
|
|
"reward": 1.0846214294433594,
|
|
"reward_std": 0.12235956788063049,
|
|
"rewards/accuracy_reward": 0.4078125,
|
|
"rewards/brier_reward": 0.7618267297744751,
|
|
"rewards/confidence_one_or_zero": 0.0013671875,
|
|
"rewards/format_reward": 0.999609375,
|
|
"rewards/mean_confidence_reward": 0.3821188509464264,
|
|
"rewards/smce": -0.06492935791611672,
|
|
"step": 75
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 819.8,
|
|
"completions/max_terminated_length": 359.6,
|
|
"completions/mean_length": 139.777734375,
|
|
"completions/mean_terminated_length": 139.50499877929687,
|
|
"completions/min_length": 62.6,
|
|
"completions/min_terminated_length": 62.6,
|
|
"epoch": 0.256,
|
|
"grad_norm": 0.0009150123805738986,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.001,
|
|
"num_tokens": 272178453.0,
|
|
"reward": 1.072606611251831,
|
|
"reward_std": 0.12022970616817474,
|
|
"rewards/accuracy_reward": 0.3763671875,
|
|
"rewards/brier_reward": 0.7690465927124024,
|
|
"rewards/confidence_one_or_zero": 0.0009765625,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4023988604545593,
|
|
"rewards/smce": -0.060530804097652435,
|
|
"step": 80
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 860.4,
|
|
"completions/max_terminated_length": 387.0,
|
|
"completions/mean_length": 140.90380859375,
|
|
"completions/mean_terminated_length": 140.63138427734376,
|
|
"completions/min_length": 65.6,
|
|
"completions/min_terminated_length": 65.6,
|
|
"epoch": 0.272,
|
|
"grad_norm": 0.0008105236338451505,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 288587004.0,
|
|
"reward": 1.081153893470764,
|
|
"reward_std": 0.11917079389095306,
|
|
"rewards/accuracy_reward": 0.3962890625,
|
|
"rewards/brier_reward": 0.7664119839668274,
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
|
"rewards/format_reward": 0.999609375,
|
|
"rewards/mean_confidence_reward": 0.40586323142051695,
|
|
"rewards/smce": -0.03432232886552811,
|
|
"step": 85
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 1096.0,
|
|
"completions/max_terminated_length": 403.2,
|
|
"completions/mean_length": 137.1322265625,
|
|
"completions/mean_terminated_length": 136.72223510742188,
|
|
"completions/min_length": 60.8,
|
|
"completions/min_terminated_length": 60.8,
|
|
"epoch": 0.288,
|
|
"grad_norm": 0.0011007882421836257,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0011,
|
|
"num_tokens": 304949414.0,
|
|
"reward": 1.0783969640731812,
|
|
"reward_std": 0.11819728463888168,
|
|
"rewards/accuracy_reward": 0.39013671875,
|
|
"rewards/brier_reward": 0.7671497702598572,
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
|
"rewards/format_reward": 0.99951171875,
|
|
"rewards/mean_confidence_reward": 0.40524903535842893,
|
|
"rewards/smce": -0.05063976794481277,
|
|
"step": 90
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 607.2,
|
|
"completions/max_terminated_length": 376.0,
|
|
"completions/mean_length": 134.31494140625,
|
|
"completions/mean_terminated_length": 134.17804565429688,
|
|
"completions/min_length": 59.4,
|
|
"completions/min_terminated_length": 59.4,
|
|
"epoch": 0.304,
|
|
"grad_norm": 0.000939876539632678,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 321254751.0,
|
|
"reward": 1.084977626800537,
|
|
"reward_std": 0.11600391864776612,
|
|
"rewards/accuracy_reward": 0.4052734375,
|
|
"rewards/brier_reward": 0.7649775862693786,
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.389298677444458,
|
|
"rewards/smce": -0.03614676035940647,
|
|
"step": 95
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 812.2,
|
|
"completions/max_terminated_length": 347.0,
|
|
"completions/mean_length": 136.36416015625,
|
|
"completions/mean_terminated_length": 136.0903289794922,
|
|
"completions/min_length": 64.6,
|
|
"completions/min_terminated_length": 64.6,
|
|
"epoch": 0.32,
|
|
"grad_norm": 0.000710589752998203,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0009,
|
|
"num_tokens": 337739824.0,
|
|
"reward": 1.0932400226593018,
|
|
"reward_std": 0.10608242303133011,
|
|
"rewards/accuracy_reward": 0.417578125,
|
|
"rewards/brier_reward": 0.7692002654075623,
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.3869974732398987,
|
|
"rewards/smce": -0.0614659234881401,
|
|
"step": 100
|
|
},
|
|
{
|
|
"epoch": 0.32,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 300.0,
|
|
"eval_completions/max_terminated_length": 300.0,
|
|
"eval_completions/mean_length": 134.9712142944336,
|
|
"eval_completions/mean_terminated_length": 134.9712142944336,
|
|
"eval_completions/min_length": 65.5,
|
|
"eval_completions/min_terminated_length": 65.5,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 337739824.0,
|
|
"eval_reward": 1.048181176185608,
|
|
"eval_reward_std": 0.19223156571388245,
|
|
"eval_rewards/accuracy_reward": 0.306640625,
|
|
"eval_rewards/brier_reward": 0.7897287011146545,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.3824804872274399,
|
|
"eval_rewards/smce": -0.07643750682473183,
|
|
"eval_runtime": 10.9079,
|
|
"eval_samples_per_second": 45.838,
|
|
"eval_steps_per_second": 0.183,
|
|
"step": 100
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 834.0,
|
|
"completions/max_terminated_length": 349.4,
|
|
"completions/mean_length": 134.5306640625,
|
|
"completions/mean_terminated_length": 134.1192596435547,
|
|
"completions/min_length": 62.2,
|
|
"completions/min_terminated_length": 62.2,
|
|
"epoch": 0.336,
|
|
"grad_norm": 0.0008598133572377264,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 353839850.0,
|
|
"reward": 1.0955848693847656,
|
|
"reward_std": 0.10709079802036285,
|
|
"rewards/accuracy_reward": 0.42373046875,
|
|
"rewards/brier_reward": 0.7679327011108399,
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
|
"rewards/format_reward": 0.99951171875,
|
|
"rewards/mean_confidence_reward": 0.3963977575302124,
|
|
"rewards/smce": -0.05861892774701118,
|
|
"step": 105
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 348.2,
|
|
"completions/max_terminated_length": 348.2,
|
|
"completions/mean_length": 139.5755859375,
|
|
"completions/mean_terminated_length": 139.5755859375,
|
|
"completions/min_length": 65.8,
|
|
"completions/min_terminated_length": 65.8,
|
|
"epoch": 0.352,
|
|
"grad_norm": 0.0007286344189196825,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 370529520.0,
|
|
"reward": 1.075069546699524,
|
|
"reward_std": 0.10592829287052155,
|
|
"rewards/accuracy_reward": 0.3759765625,
|
|
"rewards/brier_reward": 0.7741642475128174,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.3979013681411743,
|
|
"rewards/smce": -0.024860015138983727,
|
|
"step": 110
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 442.2,
|
|
"completions/max_terminated_length": 442.2,
|
|
"completions/mean_length": 146.44921875,
|
|
"completions/mean_terminated_length": 146.44921875,
|
|
"completions/min_length": 64.8,
|
|
"completions/min_terminated_length": 64.8,
|
|
"epoch": 0.368,
|
|
"grad_norm": 0.0022383604664355516,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0003,
|
|
"num_tokens": 387094632.0,
|
|
"reward": 1.088090991973877,
|
|
"reward_std": 0.11070768386125565,
|
|
"rewards/accuracy_reward": 0.40341796875,
|
|
"rewards/brier_reward": 0.7728642702102662,
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.41914639472961424,
|
|
"rewards/smce": -0.034958066418766975,
|
|
"step": 115
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 810.0,
|
|
"completions/max_terminated_length": 336.2,
|
|
"completions/mean_length": 154.13203125,
|
|
"completions/mean_terminated_length": 153.8613250732422,
|
|
"completions/min_length": 68.0,
|
|
"completions/min_terminated_length": 68.0,
|
|
"epoch": 0.384,
|
|
"grad_norm": 0.0008120255661197007,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 403529456.0,
|
|
"reward": 1.0988135099411012,
|
|
"reward_std": 0.10813373029232025,
|
|
"rewards/accuracy_reward": 0.42109375,
|
|
"rewards/brier_reward": 0.7767310023307801,
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4109645485877991,
|
|
"rewards/smce": -0.03251770362257957,
|
|
"step": 120
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 370.6,
|
|
"completions/max_terminated_length": 370.6,
|
|
"completions/mean_length": 159.397265625,
|
|
"completions/mean_terminated_length": 159.397265625,
|
|
"completions/min_length": 68.2,
|
|
"completions/min_terminated_length": 68.2,
|
|
"epoch": 0.4,
|
|
"grad_norm": 0.0007968509453348815,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 420198132.0,
|
|
"reward": 1.0878613233566283,
|
|
"reward_std": 0.11208915263414383,
|
|
"rewards/accuracy_reward": 0.3951171875,
|
|
"rewards/brier_reward": 0.7806079149246216,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.41420117020606995,
|
|
"rewards/smce": -0.03192929774522781,
|
|
"step": 125
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 841.8,
|
|
"completions/max_terminated_length": 416.2,
|
|
"completions/mean_length": 164.4671875,
|
|
"completions/mean_terminated_length": 164.1993438720703,
|
|
"completions/min_length": 74.8,
|
|
"completions/min_terminated_length": 74.8,
|
|
"epoch": 0.416,
|
|
"grad_norm": 0.0008458816446363926,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0008,
|
|
"num_tokens": 436763460.0,
|
|
"reward": 1.0922947883605958,
|
|
"reward_std": 0.11342525780200959,
|
|
"rewards/accuracy_reward": 0.4072265625,
|
|
"rewards/brier_reward": 0.7776575922966004,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.40936915278434755,
|
|
"rewards/smce": -0.023180909082293512,
|
|
"step": 130
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 851.6,
|
|
"completions/max_terminated_length": 372.2,
|
|
"completions/mean_length": 165.8568359375,
|
|
"completions/mean_terminated_length": 165.5887878417969,
|
|
"completions/min_length": 76.8,
|
|
"completions/min_terminated_length": 76.8,
|
|
"epoch": 0.432,
|
|
"grad_norm": 0.0009193225996568799,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 453476170.0,
|
|
"reward": 1.1056634187698364,
|
|
"reward_std": 0.10354232937097549,
|
|
"rewards/accuracy_reward": 0.43515625,
|
|
"rewards/brier_reward": 0.776369285583496,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.41190527081489564,
|
|
"rewards/smce": -0.042153664771467446,
|
|
"step": 135
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 496.4,
|
|
"completions/max_terminated_length": 496.4,
|
|
"completions/mean_length": 169.11396484375,
|
|
"completions/mean_terminated_length": 169.11396484375,
|
|
"completions/min_length": 78.4,
|
|
"completions/min_terminated_length": 78.4,
|
|
"epoch": 0.448,
|
|
"grad_norm": 0.0007092914893291891,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 470160697.0,
|
|
"reward": 1.0997313261032104,
|
|
"reward_std": 0.10420847237110138,
|
|
"rewards/accuracy_reward": 0.4232421875,
|
|
"rewards/brier_reward": 0.7762229681015015,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.4273776412010193,
|
|
"rewards/smce": -0.03351427987217903,
|
|
"step": 140
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 472.8,
|
|
"completions/max_terminated_length": 472.8,
|
|
"completions/mean_length": 172.30615234375,
|
|
"completions/mean_terminated_length": 172.30615234375,
|
|
"completions/min_length": 77.6,
|
|
"completions/min_terminated_length": 77.6,
|
|
"epoch": 0.464,
|
|
"grad_norm": 0.0007953299209475517,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 487095928.0,
|
|
"reward": 1.0790441513061524,
|
|
"reward_std": 0.1049978643655777,
|
|
"rewards/accuracy_reward": 0.37236328125,
|
|
"rewards/brier_reward": 0.7858292102813721,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.4317548990249634,
|
|
"rewards/smce": -0.07353323474526405,
|
|
"step": 145
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 446.8,
|
|
"completions/max_terminated_length": 446.8,
|
|
"completions/mean_length": 172.71796875,
|
|
"completions/mean_terminated_length": 172.71796875,
|
|
"completions/min_length": 78.0,
|
|
"completions/min_terminated_length": 78.0,
|
|
"epoch": 0.48,
|
|
"grad_norm": 0.000823770824354142,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 503912592.0,
|
|
"reward": 1.1081706762313843,
|
|
"reward_std": 0.10933324098587036,
|
|
"rewards/accuracy_reward": 0.44482421875,
|
|
"rewards/brier_reward": 0.7715215682983398,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.4173727512359619,
|
|
"rewards/smce": -0.05264594741165638,
|
|
"step": 150
|
|
},
|
|
{
|
|
"epoch": 0.48,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 335.0,
|
|
"eval_completions/max_terminated_length": 335.0,
|
|
"eval_completions/mean_length": 171.31009674072266,
|
|
"eval_completions/mean_terminated_length": 171.31009674072266,
|
|
"eval_completions/min_length": 89.5,
|
|
"eval_completions/min_terminated_length": 89.5,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 503912592.0,
|
|
"eval_reward": 1.0670873522758484,
|
|
"eval_reward_std": 0.21636013686656952,
|
|
"eval_rewards/accuracy_reward": 0.341796875,
|
|
"eval_rewards/brier_reward": 0.7923841774463654,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.4122070372104645,
|
|
"eval_rewards/smce": -0.07125195674598217,
|
|
"eval_runtime": 12.1739,
|
|
"eval_samples_per_second": 41.072,
|
|
"eval_steps_per_second": 0.164,
|
|
"step": 150
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 892.2,
|
|
"completions/max_terminated_length": 455.0,
|
|
"completions/mean_length": 170.50908203125,
|
|
"completions/mean_terminated_length": 170.24256896972656,
|
|
"completions/min_length": 82.2,
|
|
"completions/min_terminated_length": 82.2,
|
|
"epoch": 0.496,
|
|
"grad_norm": 0.0009170905104838312,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 520966445.0,
|
|
"reward": 1.1045333862304687,
|
|
"reward_std": 0.11436926871538163,
|
|
"rewards/accuracy_reward": 0.43828125,
|
|
"rewards/brier_reward": 0.7709831476211548,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4277272403240204,
|
|
"rewards/smce": -0.031090539321303366,
|
|
"step": 155
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 604.0,
|
|
"completions/max_terminated_length": 379.4,
|
|
"completions/mean_length": 169.0560546875,
|
|
"completions/mean_terminated_length": 168.92233581542968,
|
|
"completions/min_length": 80.6,
|
|
"completions/min_terminated_length": 80.6,
|
|
"epoch": 0.512,
|
|
"grad_norm": 0.0009231609292328358,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 537843243.0,
|
|
"reward": 1.1171007394790649,
|
|
"reward_std": 0.11715430617332459,
|
|
"rewards/accuracy_reward": 0.45810546875,
|
|
"rewards/brier_reward": 0.7761961936950683,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.44229105710983274,
|
|
"rewards/smce": -0.033351265173405406,
|
|
"step": 160
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 646.8,
|
|
"completions/max_terminated_length": 433.0,
|
|
"completions/mean_length": 170.7927734375,
|
|
"completions/mean_terminated_length": 170.65943603515626,
|
|
"completions/min_length": 85.6,
|
|
"completions/min_terminated_length": 85.6,
|
|
"epoch": 0.528,
|
|
"grad_norm": 0.0008310439880006015,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 554621697.0,
|
|
"reward": 1.1092323064804077,
|
|
"reward_std": 0.11509672254323959,
|
|
"rewards/accuracy_reward": 0.43134765625,
|
|
"rewards/brier_reward": 0.7872170209884644,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.44881221652030945,
|
|
"rewards/smce": -0.032787452824413775,
|
|
"step": 165
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00029296875,
|
|
"completions/max_length": 846.4,
|
|
"completions/max_terminated_length": 423.6,
|
|
"completions/mean_length": 170.37705078125,
|
|
"completions/mean_terminated_length": 169.9770751953125,
|
|
"completions/min_length": 81.8,
|
|
"completions/min_terminated_length": 81.8,
|
|
"epoch": 0.544,
|
|
"grad_norm": 0.0007975861080922186,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0013,
|
|
"num_tokens": 571529942.0,
|
|
"reward": 1.1260975122451782,
|
|
"reward_std": 0.11977973878383637,
|
|
"rewards/accuracy_reward": 0.482421875,
|
|
"rewards/brier_reward": 0.7700685858726501,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.4528903365135193,
|
|
"rewards/smce": -0.03396207746118307,
|
|
"step": 170
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 523.2,
|
|
"completions/max_terminated_length": 523.2,
|
|
"completions/mean_length": 167.5654296875,
|
|
"completions/mean_terminated_length": 167.5654296875,
|
|
"completions/min_length": 69.8,
|
|
"completions/min_terminated_length": 69.8,
|
|
"epoch": 0.56,
|
|
"grad_norm": 0.0008357908809557557,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 588067220.0,
|
|
"reward": 1.106752872467041,
|
|
"reward_std": 0.10880879014730453,
|
|
"rewards/accuracy_reward": 0.42197265625,
|
|
"rewards/brier_reward": 0.7916343927383422,
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.45748730301856994,
|
|
"rewards/smce": -0.044184643775224686,
|
|
"step": 175
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 394.4,
|
|
"completions/max_terminated_length": 394.4,
|
|
"completions/mean_length": 167.45751953125,
|
|
"completions/mean_terminated_length": 167.45751953125,
|
|
"completions/min_length": 78.2,
|
|
"completions/min_terminated_length": 78.2,
|
|
"epoch": 0.576,
|
|
"grad_norm": 0.0007887242827564478,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0,
|
|
"num_tokens": 604968609.0,
|
|
"reward": 1.1010312557220459,
|
|
"reward_std": 0.10108543932437897,
|
|
"rewards/accuracy_reward": 0.4220703125,
|
|
"rewards/brier_reward": 0.7800945162773132,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.4567448318004608,
|
|
"rewards/smce": -0.055632703751325605,
|
|
"step": 180
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.00068359375,
|
|
"completions/max_length": 1298.0,
|
|
"completions/max_terminated_length": 441.0,
|
|
"completions/mean_length": 165.73251953125,
|
|
"completions/mean_terminated_length": 164.7956573486328,
|
|
"completions/min_length": 74.4,
|
|
"completions/min_terminated_length": 74.4,
|
|
"epoch": 0.592,
|
|
"grad_norm": 0.0007809134549461305,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0016,
|
|
"num_tokens": 621833422.0,
|
|
"reward": 1.1107731342315674,
|
|
"reward_std": 0.10482596158981324,
|
|
"rewards/accuracy_reward": 0.4439453125,
|
|
"rewards/brier_reward": 0.7782866358757019,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99931640625,
|
|
"rewards/mean_confidence_reward": 0.46025683879852297,
|
|
"rewards/smce": -0.0295250304043293,
|
|
"step": 185
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 360.4,
|
|
"completions/max_terminated_length": 360.4,
|
|
"completions/mean_length": 162.57333984375,
|
|
"completions/mean_terminated_length": 162.57333984375,
|
|
"completions/min_length": 79.2,
|
|
"completions/min_terminated_length": 79.2,
|
|
"epoch": 0.608,
|
|
"grad_norm": 0.000834187027066946,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 638497661.0,
|
|
"reward": 1.1160224199295044,
|
|
"reward_std": 0.10795456916093826,
|
|
"rewards/accuracy_reward": 0.437890625,
|
|
"rewards/brier_reward": 0.7941561222076416,
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.45286591053009034,
|
|
"rewards/smce": -0.027559639140963554,
|
|
"step": 190
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 830.6,
|
|
"completions/max_terminated_length": 486.4,
|
|
"completions/mean_length": 165.830078125,
|
|
"completions/mean_terminated_length": 165.56271667480468,
|
|
"completions/min_length": 78.0,
|
|
"completions/min_terminated_length": 78.0,
|
|
"epoch": 0.624,
|
|
"grad_norm": 0.0009933095425367355,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 655539665.0,
|
|
"reward": 1.1106883525848388,
|
|
"reward_std": 0.1104367807507515,
|
|
"rewards/accuracy_reward": 0.433203125,
|
|
"rewards/brier_reward": 0.7883719682693482,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4572490334510803,
|
|
"rewards/smce": -0.039461223781108855,
|
|
"step": 195
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 630.0,
|
|
"completions/max_terminated_length": 416.0,
|
|
"completions/mean_length": 169.82666015625,
|
|
"completions/mean_terminated_length": 169.69326782226562,
|
|
"completions/min_length": 77.4,
|
|
"completions/min_terminated_length": 77.4,
|
|
"epoch": 0.64,
|
|
"grad_norm": 0.000809962279163301,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 672621378.0,
|
|
"reward": 1.1414735794067383,
|
|
"reward_std": 0.10489663779735565,
|
|
"rewards/accuracy_reward": 0.49716796875,
|
|
"rewards/brier_reward": 0.785882031917572,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.47141573429107664,
|
|
"rewards/smce": -0.06005169451236725,
|
|
"step": 200
|
|
},
|
|
{
|
|
"epoch": 0.64,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 363.5,
|
|
"eval_completions/max_terminated_length": 363.5,
|
|
"eval_completions/mean_length": 172.84015655517578,
|
|
"eval_completions/mean_terminated_length": 172.84015655517578,
|
|
"eval_completions/min_length": 98.0,
|
|
"eval_completions/min_terminated_length": 98.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 672621378.0,
|
|
"eval_reward": 1.0921427011489868,
|
|
"eval_reward_std": 0.2488805055618286,
|
|
"eval_rewards/accuracy_reward": 0.390625,
|
|
"eval_rewards/brier_reward": 0.7936670184135437,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.4659179747104645,
|
|
"eval_rewards/smce": -0.07549317367374897,
|
|
"eval_runtime": 12.161,
|
|
"eval_samples_per_second": 41.115,
|
|
"eval_steps_per_second": 0.164,
|
|
"step": 200
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 434.2,
|
|
"completions/max_terminated_length": 434.2,
|
|
"completions/mean_length": 170.987890625,
|
|
"completions/mean_terminated_length": 170.987890625,
|
|
"completions/min_length": 82.6,
|
|
"completions/min_terminated_length": 82.6,
|
|
"epoch": 0.656,
|
|
"grad_norm": 0.000862093351315707,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0005,
|
|
"num_tokens": 689228838.0,
|
|
"reward": 1.0985867977142334,
|
|
"reward_std": 0.10535258799791336,
|
|
"rewards/accuracy_reward": 0.4134765625,
|
|
"rewards/brier_reward": 0.7837995648384094,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.46335841417312623,
|
|
"rewards/smce": -0.05806147009134292,
|
|
"step": 205
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 604.6,
|
|
"completions/max_terminated_length": 366.0,
|
|
"completions/mean_length": 169.04951171875,
|
|
"completions/mean_terminated_length": 168.91582641601562,
|
|
"completions/min_length": 81.0,
|
|
"completions/min_terminated_length": 81.0,
|
|
"epoch": 0.672,
|
|
"grad_norm": 0.000558350991923362,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 705873345.0,
|
|
"reward": 1.1212966680526733,
|
|
"reward_std": 0.0894002914428711,
|
|
"rewards/accuracy_reward": 0.45146484375,
|
|
"rewards/brier_reward": 0.7912296891212464,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.42793651223182677,
|
|
"rewards/smce": -0.043858605064451696,
|
|
"step": 210
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 823.8,
|
|
"completions/max_terminated_length": 454.2,
|
|
"completions/mean_length": 171.4181640625,
|
|
"completions/mean_terminated_length": 171.1516357421875,
|
|
"completions/min_length": 88.0,
|
|
"completions/min_terminated_length": 88.0,
|
|
"epoch": 0.688,
|
|
"grad_norm": 0.0008224369958043098,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 722582587.0,
|
|
"reward": 1.1220870733261108,
|
|
"reward_std": 0.09683922380208969,
|
|
"rewards/accuracy_reward": 0.44853515625,
|
|
"rewards/brier_reward": 0.7959356069564819,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.43074610233306887,
|
|
"rewards/smce": -0.04505564570426941,
|
|
"step": 215
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 647.4,
|
|
"completions/max_terminated_length": 406.4,
|
|
"completions/mean_length": 177.95546875,
|
|
"completions/mean_terminated_length": 177.8226776123047,
|
|
"completions/min_length": 89.4,
|
|
"completions/min_terminated_length": 89.4,
|
|
"epoch": 0.704,
|
|
"grad_norm": 0.000638193916529417,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 739270995.0,
|
|
"reward": 1.132350206375122,
|
|
"reward_std": 0.09680995047092437,
|
|
"rewards/accuracy_reward": 0.474609375,
|
|
"rewards/brier_reward": 0.7901916623115539,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.44232421517372134,
|
|
"rewards/smce": -0.03818157017230987,
|
|
"step": 220
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 654.8,
|
|
"completions/max_terminated_length": 429.0,
|
|
"completions/mean_length": 181.17880859375,
|
|
"completions/mean_terminated_length": 181.0465118408203,
|
|
"completions/min_length": 90.6,
|
|
"completions/min_terminated_length": 90.6,
|
|
"epoch": 0.72,
|
|
"grad_norm": 0.0008984857704490423,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 756136122.0,
|
|
"reward": 1.1432342529296875,
|
|
"reward_std": 0.09815936386585236,
|
|
"rewards/accuracy_reward": 0.48505859375,
|
|
"rewards/brier_reward": 0.8015105724334717,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.48147950172424314,
|
|
"rewards/smce": -0.03847266510128975,
|
|
"step": 225
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 691.4,
|
|
"completions/max_terminated_length": 456.6,
|
|
"completions/mean_length": 181.17685546875,
|
|
"completions/mean_terminated_length": 181.04478759765624,
|
|
"completions/min_length": 90.4,
|
|
"completions/min_terminated_length": 90.4,
|
|
"epoch": 0.736,
|
|
"grad_norm": 0.0009152474231086671,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 772930957.0,
|
|
"reward": 1.1385881662368775,
|
|
"reward_std": 0.10552089065313339,
|
|
"rewards/accuracy_reward": 0.4880859375,
|
|
"rewards/brier_reward": 0.789192795753479,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.5017031252384185,
|
|
"rewards/smce": -0.057008542120456696,
|
|
"step": 230
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 408.0,
|
|
"completions/max_terminated_length": 408.0,
|
|
"completions/mean_length": 182.5458984375,
|
|
"completions/mean_terminated_length": 182.5458984375,
|
|
"completions/min_length": 89.4,
|
|
"completions/min_terminated_length": 89.4,
|
|
"epoch": 0.752,
|
|
"grad_norm": 0.0009231127914972603,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 790027427.0,
|
|
"reward": 1.1306201219558716,
|
|
"reward_std": 0.10703302025794983,
|
|
"rewards/accuracy_reward": 0.47080078125,
|
|
"rewards/brier_reward": 0.79044269323349,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.480672562122345,
|
|
"rewards/smce": -0.04172721914947033,
|
|
"step": 235
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 702.4,
|
|
"completions/max_terminated_length": 497.4,
|
|
"completions/mean_length": 189.3001953125,
|
|
"completions/mean_terminated_length": 189.16878356933594,
|
|
"completions/min_length": 95.6,
|
|
"completions/min_terminated_length": 95.6,
|
|
"epoch": 0.768,
|
|
"grad_norm": 0.0008118224795907736,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0001,
|
|
"num_tokens": 806898565.0,
|
|
"reward": 1.1371404886245728,
|
|
"reward_std": 0.10227844417095185,
|
|
"rewards/accuracy_reward": 0.4744140625,
|
|
"rewards/brier_reward": 0.7999661445617676,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.4725029289722443,
|
|
"rewards/smce": -0.02483233194798231,
|
|
"step": 240
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 443.4,
|
|
"completions/max_terminated_length": 443.4,
|
|
"completions/mean_length": 187.94228515625,
|
|
"completions/mean_terminated_length": 187.94228515625,
|
|
"completions/min_length": 97.0,
|
|
"completions/min_terminated_length": 97.0,
|
|
"epoch": 0.784,
|
|
"grad_norm": 0.0009792657801881433,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 823997462.0,
|
|
"reward": 1.1333154678344726,
|
|
"reward_std": 0.10855860561132431,
|
|
"rewards/accuracy_reward": 0.47734375,
|
|
"rewards/brier_reward": 0.7892932295799255,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.49031250476837157,
|
|
"rewards/smce": -0.06904417928308249,
|
|
"step": 245
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 656.8,
|
|
"completions/max_terminated_length": 427.0,
|
|
"completions/mean_length": 186.925,
|
|
"completions/mean_terminated_length": 186.79339599609375,
|
|
"completions/min_length": 92.0,
|
|
"completions/min_terminated_length": 92.0,
|
|
"epoch": 0.8,
|
|
"grad_norm": 0.0009143622010014951,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 840922134.0,
|
|
"reward": 1.1496532440185547,
|
|
"reward_std": 0.1164577692747116,
|
|
"rewards/accuracy_reward": 0.50009765625,
|
|
"rewards/brier_reward": 0.7993093371391297,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.5264248371124267,
|
|
"rewards/smce": -0.03849135749042034,
|
|
"step": 250
|
|
},
|
|
{
|
|
"epoch": 0.8,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 372.0,
|
|
"eval_completions/max_terminated_length": 372.0,
|
|
"eval_completions/mean_length": 188.12112426757812,
|
|
"eval_completions/mean_terminated_length": 188.12112426757812,
|
|
"eval_completions/min_length": 123.0,
|
|
"eval_completions/min_terminated_length": 123.0,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 840922134.0,
|
|
"eval_reward": 1.0896968841552734,
|
|
"eval_reward_std": 0.2757375240325928,
|
|
"eval_rewards/accuracy_reward": 0.390625,
|
|
"eval_rewards/brier_reward": 0.7887792587280273,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.5048828274011612,
|
|
"eval_rewards/smce": -0.11445313319563866,
|
|
"eval_runtime": 12.1809,
|
|
"eval_samples_per_second": 41.048,
|
|
"eval_steps_per_second": 0.164,
|
|
"step": 250
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 644.8,
|
|
"completions/max_terminated_length": 422.8,
|
|
"completions/mean_length": 183.82333984375,
|
|
"completions/mean_terminated_length": 183.69131469726562,
|
|
"completions/min_length": 95.8,
|
|
"completions/min_terminated_length": 95.8,
|
|
"epoch": 0.816,
|
|
"grad_norm": 0.0011780121130868793,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0002,
|
|
"num_tokens": 857903653.0,
|
|
"reward": 1.1443308353424073,
|
|
"reward_std": 0.12021788060665131,
|
|
"rewards/accuracy_reward": 0.48671875,
|
|
"rewards/brier_reward": 0.8022388696670533,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.5260625123977661,
|
|
"rewards/smce": -0.040660728514194486,
|
|
"step": 255
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 651.4,
|
|
"completions/max_terminated_length": 417.2,
|
|
"completions/mean_length": 189.85849609375,
|
|
"completions/mean_terminated_length": 189.72706298828126,
|
|
"completions/min_length": 88.8,
|
|
"completions/min_terminated_length": 88.8,
|
|
"epoch": 0.832,
|
|
"grad_norm": 0.0010882235364988446,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 874856156.0,
|
|
"reward": 1.150579810142517,
|
|
"reward_std": 0.10773497819900513,
|
|
"rewards/accuracy_reward": 0.49453125,
|
|
"rewards/brier_reward": 0.8069238066673279,
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.49002930521965027,
|
|
"rewards/smce": -0.03444872796535492,
|
|
"step": 260
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 889.6,
|
|
"completions/max_terminated_length": 454.0,
|
|
"completions/mean_length": 188.8404296875,
|
|
"completions/mean_terminated_length": 188.57715454101563,
|
|
"completions/min_length": 101.4,
|
|
"completions/min_terminated_length": 101.4,
|
|
"epoch": 0.848,
|
|
"grad_norm": 0.0009498689323663712,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0006,
|
|
"num_tokens": 891804250.0,
|
|
"reward": 1.1383338212966918,
|
|
"reward_std": 0.10923963636159897,
|
|
"rewards/accuracy_reward": 0.47392578125,
|
|
"rewards/brier_reward": 0.8029386281967164,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4741074204444885,
|
|
"rewards/smce": -0.023679725266993044,
|
|
"step": 265
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 745.0,
|
|
"completions/max_terminated_length": 551.8,
|
|
"completions/mean_length": 194.55390625,
|
|
"completions/mean_terminated_length": 194.42296447753907,
|
|
"completions/min_length": 102.4,
|
|
"completions/min_terminated_length": 102.4,
|
|
"epoch": 0.864,
|
|
"grad_norm": 0.0009577281307429075,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 908783298.0,
|
|
"reward": 1.1508663177490235,
|
|
"reward_std": 0.1009971633553505,
|
|
"rewards/accuracy_reward": 0.5048828125,
|
|
"rewards/brier_reward": 0.7971471190452576,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99970703125,
|
|
"rewards/mean_confidence_reward": 0.45706055760383607,
|
|
"rewards/smce": -0.051518266275525096,
|
|
"step": 270
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 486.0,
|
|
"completions/max_terminated_length": 486.0,
|
|
"completions/mean_length": 197.091015625,
|
|
"completions/mean_terminated_length": 197.091015625,
|
|
"completions/min_length": 104.2,
|
|
"completions/min_terminated_length": 104.2,
|
|
"epoch": 0.88,
|
|
"grad_norm": 0.0008310083649121225,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 925948582.0,
|
|
"reward": 1.1340973615646361,
|
|
"reward_std": 0.09887916445732117,
|
|
"rewards/accuracy_reward": 0.45927734375,
|
|
"rewards/brier_reward": 0.8090185642242431,
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.48420898914337157,
|
|
"rewards/smce": -0.044120116904377935,
|
|
"step": 275
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 658.2,
|
|
"completions/max_terminated_length": 441.4,
|
|
"completions/mean_length": 198.308203125,
|
|
"completions/mean_terminated_length": 198.17782897949218,
|
|
"completions/min_length": 103.6,
|
|
"completions/min_terminated_length": 103.6,
|
|
"epoch": 0.896,
|
|
"grad_norm": 0.0007895663729868829,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0004,
|
|
"num_tokens": 943090106.0,
|
|
"reward": 1.138153624534607,
|
|
"reward_std": 0.10331566035747528,
|
|
"rewards/accuracy_reward": 0.4728515625,
|
|
"rewards/brier_reward": 0.8035562992095947,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.49338183999061586,
|
|
"rewards/smce": -0.038932828418910506,
|
|
"step": 280
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0001953125,
|
|
"completions/max_length": 862.8,
|
|
"completions/max_terminated_length": 424.4,
|
|
"completions/mean_length": 196.6095703125,
|
|
"completions/mean_terminated_length": 196.3483428955078,
|
|
"completions/min_length": 101.2,
|
|
"completions/min_terminated_length": 101.2,
|
|
"epoch": 0.912,
|
|
"grad_norm": 0.000810799770988524,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0007,
|
|
"num_tokens": 960154684.0,
|
|
"reward": 1.1308280229568481,
|
|
"reward_std": 0.09144939780235291,
|
|
"rewards/accuracy_reward": 0.46455078125,
|
|
"rewards/brier_reward": 0.7973055481910706,
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
|
"rewards/format_reward": 0.9998046875,
|
|
"rewards/mean_confidence_reward": 0.4340830147266388,
|
|
"rewards/smce": -0.0579361766576767,
|
|
"step": 285
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 403.4,
|
|
"completions/max_terminated_length": 403.4,
|
|
"completions/mean_length": 191.276171875,
|
|
"completions/mean_terminated_length": 191.276171875,
|
|
"completions/min_length": 103.6,
|
|
"completions/min_terminated_length": 103.6,
|
|
"epoch": 0.928,
|
|
"grad_norm": 0.0007378010777756572,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0003,
|
|
"num_tokens": 977140168.0,
|
|
"reward": 1.140564775466919,
|
|
"reward_std": 0.08266374170780182,
|
|
"rewards/accuracy_reward": 0.4955078125,
|
|
"rewards/brier_reward": 0.7856297492980957,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.4091914176940918,
|
|
"rewards/smce": -0.08774990439414979,
|
|
"step": 290
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 429.0,
|
|
"completions/max_terminated_length": 429.0,
|
|
"completions/mean_length": 188.0517578125,
|
|
"completions/mean_terminated_length": 188.0517578125,
|
|
"completions/min_length": 98.4,
|
|
"completions/min_terminated_length": 98.4,
|
|
"epoch": 0.944,
|
|
"grad_norm": 0.0008984626037999988,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 994041242.0,
|
|
"reward": 1.1434009790420532,
|
|
"reward_std": 0.09434707462787628,
|
|
"rewards/accuracy_reward": 0.4908203125,
|
|
"rewards/brier_reward": 0.7959874987602233,
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.45385352373123167,
|
|
"rewards/smce": -0.06694994941353798,
|
|
"step": 295
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 452.2,
|
|
"completions/max_terminated_length": 452.2,
|
|
"completions/mean_length": 188.75078125,
|
|
"completions/mean_terminated_length": 188.75078125,
|
|
"completions/min_length": 101.8,
|
|
"completions/min_terminated_length": 101.8,
|
|
"epoch": 0.96,
|
|
"grad_norm": 0.0009762116824276745,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0002,
|
|
"num_tokens": 1010914370.0,
|
|
"reward": 1.1328925371170044,
|
|
"reward_std": 0.09758208692073822,
|
|
"rewards/accuracy_reward": 0.46142578125,
|
|
"rewards/brier_reward": 0.8043647527694702,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.5251660048961639,
|
|
"rewards/smce": -0.06499951593577862,
|
|
"step": 300
|
|
},
|
|
{
|
|
"epoch": 0.96,
|
|
"eval_completions/clipped_ratio": 0.0,
|
|
"eval_completions/max_length": 424.0,
|
|
"eval_completions/max_terminated_length": 424.0,
|
|
"eval_completions/mean_length": 195.71150970458984,
|
|
"eval_completions/mean_terminated_length": 195.71150970458984,
|
|
"eval_completions/min_length": 109.5,
|
|
"eval_completions/min_terminated_length": 109.5,
|
|
"eval_loss": 0.0,
|
|
"eval_num_tokens": 1010914370.0,
|
|
"eval_reward": 1.0907545685768127,
|
|
"eval_reward_std": 0.29445892572402954,
|
|
"eval_rewards/accuracy_reward": 0.39453125,
|
|
"eval_rewards/brier_reward": 0.7869921922683716,
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
|
"eval_rewards/format_reward": 1.0,
|
|
"eval_rewards/mean_confidence_reward": 0.5476562678813934,
|
|
"eval_rewards/smce": -0.15322265774011612,
|
|
"eval_runtime": 13.5384,
|
|
"eval_samples_per_second": 36.932,
|
|
"eval_steps_per_second": 0.148,
|
|
"step": 300
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.0,
|
|
"completions/max_length": 411.8,
|
|
"completions/max_terminated_length": 411.8,
|
|
"completions/mean_length": 198.3265625,
|
|
"completions/mean_terminated_length": 198.3265625,
|
|
"completions/min_length": 103.0,
|
|
"completions/min_terminated_length": 103.0,
|
|
"epoch": 0.976,
|
|
"grad_norm": 0.0008806736441329122,
|
|
"learning_rate": 1e-06,
|
|
"loss": -0.0001,
|
|
"num_tokens": 1027806354.0,
|
|
"reward": 1.149838137626648,
|
|
"reward_std": 0.10467096716165543,
|
|
"rewards/accuracy_reward": 0.4970703125,
|
|
"rewards/brier_reward": 0.8026108384132385,
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
|
"rewards/format_reward": 1.0,
|
|
"rewards/mean_confidence_reward": 0.5535937666893005,
|
|
"rewards/smce": -0.05930444747209549,
|
|
"step": 305
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 9.765625e-05,
|
|
"completions/max_length": 701.0,
|
|
"completions/max_terminated_length": 556.0,
|
|
"completions/mean_length": 209.83701171875,
|
|
"completions/mean_terminated_length": 209.70668029785156,
|
|
"completions/min_length": 115.0,
|
|
"completions/min_terminated_length": 115.0,
|
|
"epoch": 0.992,
|
|
"grad_norm": 0.0007852715207263827,
|
|
"learning_rate": 1e-06,
|
|
"loss": 0.0,
|
|
"num_tokens": 1045083565.0,
|
|
"reward": 1.1392642974853515,
|
|
"reward_std": 0.11366586685180664,
|
|
"rewards/accuracy_reward": 0.483203125,
|
|
"rewards/brier_reward": 0.7954272270202637,
|
|
"rewards/confidence_one_or_zero": 0.00029296875,
|
|
"rewards/format_reward": 0.99990234375,
|
|
"rewards/mean_confidence_reward": 0.500665032863617,
|
|
"rewards/smce": -0.049921296536922455,
|
|
"step": 310
|
|
},
|
|
{
|
|
"clip_ratio/high_max": 0.0,
|
|
"clip_ratio/high_mean": 0.0,
|
|
"clip_ratio/low_mean": 0.0,
|
|
"clip_ratio/low_min": 0.0,
|
|
"clip_ratio/region_mean": 0.0,
|
|
"completions/clipped_ratio": 0.000244140625,
|
|
"completions/max_length": 996.0,
|
|
"completions/max_terminated_length": 477.0,
|
|
"completions/mean_length": 218.3510284423828,
|
|
"completions/mean_terminated_length": 218.0293197631836,
|
|
"completions/min_length": 115.5,
|
|
"completions/min_terminated_length": 115.5,
|
|
"epoch": 0.9984,
|
|
"num_tokens": 1051984536.0,
|
|
"reward": 1.1309390664100647,
|
|
"reward_std": 0.1034131683409214,
|
|
"rewards/accuracy_reward": 0.474609375,
|
|
"rewards/brier_reward": 0.7875158786773682,
|
|
"rewards/confidence_one_or_zero": 0.0,
|
|
"rewards/format_reward": 0.999755859375,
|
|
"rewards/mean_confidence_reward": 0.509765625,
|
|
"rewards/smce": -0.03950684051960707,
|
|
"step": 312,
|
|
"total_flos": 0.0,
|
|
"train_loss": 0.004247323407187837,
|
|
"train_runtime": 42843.6506,
|
|
"train_samples_per_second": 0.467,
|
|
"train_steps_per_second": 0.007
|
|
}
|
|
],
|
|
"logging_steps": 5,
|
|
"max_steps": 312,
|
|
"num_input_tokens_seen": 1051984536,
|
|
"num_train_epochs": 1,
|
|
"save_steps": 60,
|
|
"stateful_callbacks": {
|
|
"TrainerControl": {
|
|
"args": {
|
|
"should_epoch_stop": false,
|
|
"should_evaluate": false,
|
|
"should_log": false,
|
|
"should_save": true,
|
|
"should_training_stop": true
|
|
},
|
|
"attributes": {}
|
|
}
|
|
},
|
|
"total_flos": 0.0,
|
|
"train_batch_size": 4,
|
|
"trial_name": null,
|
|
"trial_params": null
|
|
}
|