1944 lines
72 KiB
JSON
1944 lines
72 KiB
JSON
|
{
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"eval_steps": 50,
|
||
|
|
"global_step": 312,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.1111328125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1527.8,
|
||
|
|
"completions/mean_length": 468.4916015625,
|
||
|
|
"completions/mean_terminated_length": 335.114697265625,
|
||
|
|
"completions/min_length": 1.0,
|
||
|
|
"completions/min_terminated_length": 1.0,
|
||
|
|
"epoch": 0.016,
|
||
|
|
"grad_norm": 0.004000246990472078,
|
||
|
|
"learning_rate": 3.1249999999999997e-07,
|
||
|
|
"loss": 0.0437,
|
||
|
|
"num_tokens": 19740202.0,
|
||
|
|
"reward": 0.20192502439022064,
|
||
|
|
"reward_std": 0.35805765390396116,
|
||
|
|
"rewards/accuracy_reward": 0.04697265625,
|
||
|
|
"rewards/brier_reward": 0.09627992808818817,
|
||
|
|
"rewards/confidence_one_or_zero": 0.40458984375,
|
||
|
|
"rewards/format_reward": 0.26064453125,
|
||
|
|
"rewards/mean_confidence_reward": 0.536387276649475,
|
||
|
|
"rewards/smce": -0.4894861876964569,
|
||
|
|
"step": 5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.09931640625,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1529.2,
|
||
|
|
"completions/mean_length": 433.90341796875,
|
||
|
|
"completions/mean_terminated_length": 312.67100830078124,
|
||
|
|
"completions/min_length": 1.2,
|
||
|
|
"completions/min_terminated_length": 1.2,
|
||
|
|
"epoch": 0.032,
|
||
|
|
"grad_norm": 0.004486603196710348,
|
||
|
|
"learning_rate": 6.249999999999999e-07,
|
||
|
|
"loss": 0.05,
|
||
|
|
"num_tokens": 39237421.0,
|
||
|
|
"reward": 0.2321306884288788,
|
||
|
|
"reward_std": 0.38784984946250917,
|
||
|
|
"rewards/accuracy_reward": 0.06201171875,
|
||
|
|
"rewards/brier_reward": 0.1105025827884674,
|
||
|
|
"rewards/confidence_one_or_zero": 0.44853515625,
|
||
|
|
"rewards/format_reward": 0.291796875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5800010561943054,
|
||
|
|
"rewards/smce": -0.5183212876319885,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0544921875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1513.8,
|
||
|
|
"completions/mean_length": 315.594140625,
|
||
|
|
"completions/mean_terminated_length": 245.67587585449218,
|
||
|
|
"completions/min_length": 2.4,
|
||
|
|
"completions/min_terminated_length": 2.4,
|
||
|
|
"epoch": 0.048,
|
||
|
|
"grad_norm": 0.005046067293733358,
|
||
|
|
"learning_rate": 9.374999999999999e-07,
|
||
|
|
"loss": 0.0569,
|
||
|
|
"num_tokens": 57570577.0,
|
||
|
|
"reward": 0.4244216620922089,
|
||
|
|
"reward_std": 0.4531022846698761,
|
||
|
|
"rewards/accuracy_reward": 0.12109375,
|
||
|
|
"rewards/brier_reward": 0.2120816856622696,
|
||
|
|
"rewards/confidence_one_or_zero": 0.5412109375,
|
||
|
|
"rewards/format_reward": 0.51572265625,
|
||
|
|
"rewards/mean_confidence_reward": 0.691961121559143,
|
||
|
|
"rewards/smce": -0.572067379951477,
|
||
|
|
"step": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0216796875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1420.8,
|
||
|
|
"completions/mean_length": 208.04306640625,
|
||
|
|
"completions/mean_terminated_length": 178.70152587890624,
|
||
|
|
"completions/min_length": 14.6,
|
||
|
|
"completions/min_terminated_length": 14.6,
|
||
|
|
"epoch": 0.064,
|
||
|
|
"grad_norm": 0.002767345402389765,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0373,
|
||
|
|
"num_tokens": 74636170.0,
|
||
|
|
"reward": 0.6850485920906066,
|
||
|
|
"reward_std": 0.394974547624588,
|
||
|
|
"rewards/accuracy_reward": 0.1884765625,
|
||
|
|
"rewards/brier_reward": 0.33910041451454165,
|
||
|
|
"rewards/confidence_one_or_zero": 0.5734375,
|
||
|
|
"rewards/format_reward": 0.842578125,
|
||
|
|
"rewards/mean_confidence_reward": 0.7928766369819641,
|
||
|
|
"rewards/smce": -0.6067353248596191,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.01435546875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1331.4,
|
||
|
|
"completions/mean_length": 163.50029296875,
|
||
|
|
"completions/mean_terminated_length": 143.52745666503907,
|
||
|
|
"completions/min_length": 26.2,
|
||
|
|
"completions/min_terminated_length": 26.2,
|
||
|
|
"epoch": 0.08,
|
||
|
|
"grad_norm": 0.00286945141851902,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.026,
|
||
|
|
"num_tokens": 91538797.0,
|
||
|
|
"reward": 0.7999858379364013,
|
||
|
|
"reward_std": 0.32701762318611144,
|
||
|
|
"rewards/accuracy_reward": 0.2197265625,
|
||
|
|
"rewards/brier_reward": 0.42199711203575135,
|
||
|
|
"rewards/confidence_one_or_zero": 0.477734375,
|
||
|
|
"rewards/format_reward": 0.95830078125,
|
||
|
|
"rewards/mean_confidence_reward": 0.7663683414459228,
|
||
|
|
"rewards/smce": -0.5526032984256745,
|
||
|
|
"step": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0083984375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 1194.4,
|
||
|
|
"completions/mean_length": 137.20341796875,
|
||
|
|
"completions/mean_terminated_length": 125.36412811279297,
|
||
|
|
"completions/min_length": 35.4,
|
||
|
|
"completions/min_terminated_length": 35.4,
|
||
|
|
"epoch": 0.096,
|
||
|
|
"grad_norm": 0.0021565118804574013,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0144,
|
||
|
|
"num_tokens": 108040208.0,
|
||
|
|
"reward": 0.9100030303001404,
|
||
|
|
"reward_std": 0.2945687472820282,
|
||
|
|
"rewards/accuracy_reward": 0.27177734375,
|
||
|
|
"rewards/brier_reward": 0.5665248811244965,
|
||
|
|
"rewards/confidence_one_or_zero": 0.31298828125,
|
||
|
|
"rewards/format_reward": 0.98173828125,
|
||
|
|
"rewards/mean_confidence_reward": 0.618563997745514,
|
||
|
|
"rewards/smce": -0.36314412355422976,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00263671875,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 791.4,
|
||
|
|
"completions/mean_length": 119.57265625,
|
||
|
|
"completions/mean_terminated_length": 115.82948455810546,
|
||
|
|
"completions/min_length": 45.8,
|
||
|
|
"completions/min_terminated_length": 45.8,
|
||
|
|
"epoch": 0.112,
|
||
|
|
"grad_norm": 0.0013891329290345311,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0064,
|
||
|
|
"num_tokens": 124112504.0,
|
||
|
|
"reward": 0.9935638666152954,
|
||
|
|
"reward_std": 0.20782245695590973,
|
||
|
|
"rewards/accuracy_reward": 0.2794921875,
|
||
|
|
"rewards/brier_reward": 0.7140938878059387,
|
||
|
|
"rewards/confidence_one_or_zero": 0.18212890625,
|
||
|
|
"rewards/format_reward": 0.9935546875,
|
||
|
|
"rewards/mean_confidence_reward": 0.3873016178607941,
|
||
|
|
"rewards/smce": -0.14182758033275605,
|
||
|
|
"step": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00146484375,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 668.4,
|
||
|
|
"completions/mean_length": 116.70654296875,
|
||
|
|
"completions/mean_terminated_length": 114.6251708984375,
|
||
|
|
"completions/min_length": 28.0,
|
||
|
|
"completions/min_terminated_length": 28.0,
|
||
|
|
"epoch": 0.128,
|
||
|
|
"grad_norm": 0.0009953748667612672,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0026,
|
||
|
|
"num_tokens": 140292219.0,
|
||
|
|
"reward": 1.029345941543579,
|
||
|
|
"reward_std": 0.14419816434383392,
|
||
|
|
"rewards/accuracy_reward": 0.30634765625,
|
||
|
|
"rewards/brier_reward": 0.7557683944702148,
|
||
|
|
"rewards/confidence_one_or_zero": 0.13935546875,
|
||
|
|
"rewards/format_reward": 0.99658203125,
|
||
|
|
"rewards/mean_confidence_reward": 0.27119520902633665,
|
||
|
|
"rewards/smce": -0.07006132453680039,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00078125,
|
||
|
|
"completions/max_length": 1536.0,
|
||
|
|
"completions/max_terminated_length": 606.0,
|
||
|
|
"completions/mean_length": 115.9509765625,
|
||
|
|
"completions/mean_terminated_length": 114.84084930419922,
|
||
|
|
"completions/min_length": 49.8,
|
||
|
|
"completions/min_terminated_length": 49.8,
|
||
|
|
"epoch": 0.144,
|
||
|
|
"grad_norm": 0.0008894961792975664,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"num_tokens": 156452069.0,
|
||
|
|
"reward": 1.0353077411651612,
|
||
|
|
"reward_std": 0.10861439108848572,
|
||
|
|
"rewards/accuracy_reward": 0.28837890625,
|
||
|
|
"rewards/brier_reward": 0.7841946363449097,
|
||
|
|
"rewards/confidence_one_or_zero": 0.07890625,
|
||
|
|
"rewards/format_reward": 0.998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.24555768072605133,
|
||
|
|
"rewards/smce": -0.05660989247262478,
|
||
|
|
"step": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00087890625,
|
||
|
|
"completions/max_length": 1308.0,
|
||
|
|
"completions/max_terminated_length": 471.6,
|
||
|
|
"completions/mean_length": 117.7646484375,
|
||
|
|
"completions/mean_terminated_length": 116.51729736328124,
|
||
|
|
"completions/min_length": 50.6,
|
||
|
|
"completions/min_terminated_length": 50.6,
|
||
|
|
"epoch": 0.16,
|
||
|
|
"grad_norm": 0.0008676417637616396,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0021,
|
||
|
|
"num_tokens": 172609243.0,
|
||
|
|
"reward": 1.0493716478347779,
|
||
|
|
"reward_std": 0.10434378683567047,
|
||
|
|
"rewards/accuracy_reward": 0.33759765625,
|
||
|
|
"rewards/brier_reward": 0.7627172112464905,
|
||
|
|
"rewards/confidence_one_or_zero": 0.04599609375,
|
||
|
|
"rewards/format_reward": 0.9984375,
|
||
|
|
"rewards/mean_confidence_reward": 0.24653425812721252,
|
||
|
|
"rewards/smce": -0.09616437256336212,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.16,
|
||
|
|
"eval_completions/clipped_ratio": 0.002049180327868827,
|
||
|
|
"eval_completions/max_length": 906.0,
|
||
|
|
"eval_completions/max_terminated_length": 298.0,
|
||
|
|
"eval_completions/mean_length": 121.82882690429688,
|
||
|
|
"eval_completions/mean_terminated_length": 118.9295425415039,
|
||
|
|
"eval_completions/min_length": 62.5,
|
||
|
|
"eval_completions/min_terminated_length": 62.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 172609243.0,
|
||
|
|
"eval_reward": 1.026913046836853,
|
||
|
|
"eval_reward_std": 0.15706007927656174,
|
||
|
|
"eval_rewards/accuracy_reward": 0.26171875,
|
||
|
|
"eval_rewards/brier_reward": 0.7960156202316284,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.03125,
|
||
|
|
"eval_rewards/format_reward": 0.99609375,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.2691406309604645,
|
||
|
|
"eval_rewards/smce": -0.026340818963944912,
|
||
|
|
"eval_runtime": 21.7347,
|
||
|
|
"eval_samples_per_second": 23.005,
|
||
|
|
"eval_steps_per_second": 0.092,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 854.4,
|
||
|
|
"completions/max_terminated_length": 403.8,
|
||
|
|
"completions/mean_length": 119.148828125,
|
||
|
|
"completions/mean_terminated_length": 118.87189025878907,
|
||
|
|
"completions/min_length": 54.4,
|
||
|
|
"completions/min_terminated_length": 54.4,
|
||
|
|
"epoch": 0.176,
|
||
|
|
"grad_norm": 0.0009692950989119709,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 188849903.0,
|
||
|
|
"reward": 1.0605363368988037,
|
||
|
|
"reward_std": 0.10660427510738373,
|
||
|
|
"rewards/accuracy_reward": 0.36181640625,
|
||
|
|
"rewards/brier_reward": 0.7598503589630127,
|
||
|
|
"rewards/confidence_one_or_zero": 0.02255859375,
|
||
|
|
"rewards/format_reward": 0.9994140625,
|
||
|
|
"rewards/mean_confidence_reward": 0.2778467535972595,
|
||
|
|
"rewards/smce": -0.08778789937496186,
|
||
|
|
"step": 55
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00068359375,
|
||
|
|
"completions/max_length": 1057.4,
|
||
|
|
"completions/max_terminated_length": 327.0,
|
||
|
|
"completions/mean_length": 122.08525390625,
|
||
|
|
"completions/mean_terminated_length": 121.11732788085938,
|
||
|
|
"completions/min_length": 56.4,
|
||
|
|
"completions/min_terminated_length": 56.4,
|
||
|
|
"epoch": 0.192,
|
||
|
|
"grad_norm": 0.0009788707830011845,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"num_tokens": 205073144.0,
|
||
|
|
"reward": 1.0588705778121947,
|
||
|
|
"reward_std": 0.10916559845209121,
|
||
|
|
"rewards/accuracy_reward": 0.35341796875,
|
||
|
|
"rewards/brier_reward": 0.7655003070831299,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0072265625,
|
||
|
|
"rewards/format_reward": 0.998828125,
|
||
|
|
"rewards/mean_confidence_reward": 0.31695576310157775,
|
||
|
|
"rewards/smce": -0.0588741596788168,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 944.6,
|
||
|
|
"completions/max_terminated_length": 482.0,
|
||
|
|
"completions/mean_length": 124.083984375,
|
||
|
|
"completions/mean_terminated_length": 123.80798950195313,
|
||
|
|
"completions/min_length": 58.8,
|
||
|
|
"completions/min_terminated_length": 58.8,
|
||
|
|
"epoch": 0.208,
|
||
|
|
"grad_norm": 0.001055966247804463,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 221389876.0,
|
||
|
|
"reward": 1.0805248260498046,
|
||
|
|
"reward_std": 0.12115304917097092,
|
||
|
|
"rewards/accuracy_reward": 0.40693359375,
|
||
|
|
"rewards/brier_reward": 0.7546101808547974,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00390625,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.36039193868637087,
|
||
|
|
"rewards/smce": -0.06622847616672516,
|
||
|
|
"step": 65
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 823.0,
|
||
|
|
"completions/max_terminated_length": 331.6,
|
||
|
|
"completions/mean_length": 126.92802734375,
|
||
|
|
"completions/mean_terminated_length": 126.65286102294922,
|
||
|
|
"completions/min_length": 59.0,
|
||
|
|
"completions/min_terminated_length": 59.0,
|
||
|
|
"epoch": 0.224,
|
||
|
|
"grad_norm": 0.0010693792719393969,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 237640691.0,
|
||
|
|
"reward": 1.0558359622955322,
|
||
|
|
"reward_std": 0.11917430013418198,
|
||
|
|
"rewards/accuracy_reward": 0.33271484375,
|
||
|
|
"rewards/brier_reward": 0.7793541669845581,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0029296875,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.3942097663879395,
|
||
|
|
"rewards/smce": -0.07278793528676034,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 815.0,
|
||
|
|
"completions/max_terminated_length": 355.2,
|
||
|
|
"completions/mean_length": 130.259765625,
|
||
|
|
"completions/mean_terminated_length": 129.98523559570313,
|
||
|
|
"completions/min_length": 60.8,
|
||
|
|
"completions/min_terminated_length": 60.8,
|
||
|
|
"epoch": 0.24,
|
||
|
|
"grad_norm": 0.0009581490885466337,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 253971927.0,
|
||
|
|
"reward": 1.060979390144348,
|
||
|
|
"reward_std": 0.11544482111930847,
|
||
|
|
"rewards/accuracy_reward": 0.34765625,
|
||
|
|
"rewards/brier_reward": 0.7750885605812072,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0015625,
|
||
|
|
"rewards/format_reward": 0.99921875,
|
||
|
|
"rewards/mean_confidence_reward": 0.3995449125766754,
|
||
|
|
"rewards/smce": -0.05598033778369427,
|
||
|
|
"step": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 591.4,
|
||
|
|
"completions/max_terminated_length": 350.0,
|
||
|
|
"completions/mean_length": 131.16630859375,
|
||
|
|
"completions/mean_terminated_length": 131.029052734375,
|
||
|
|
"completions/min_length": 47.8,
|
||
|
|
"completions/min_terminated_length": 47.8,
|
||
|
|
"epoch": 0.256,
|
||
|
|
"grad_norm": 0.0009065366466529667,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 270476318.0,
|
||
|
|
"reward": 1.0496712446212768,
|
||
|
|
"reward_std": 0.11388081759214401,
|
||
|
|
"rewards/accuracy_reward": 0.3197265625,
|
||
|
|
"rewards/brier_reward": 0.7800132632255554,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00078125,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.38173535466194153,
|
||
|
|
"rewards/smce": -0.0739667683839798,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 344.2,
|
||
|
|
"completions/max_terminated_length": 344.2,
|
||
|
|
"completions/mean_length": 133.273046875,
|
||
|
|
"completions/mean_terminated_length": 133.273046875,
|
||
|
|
"completions/min_length": 61.0,
|
||
|
|
"completions/min_terminated_length": 61.0,
|
||
|
|
"epoch": 0.272,
|
||
|
|
"grad_norm": 0.000878436432685703,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 286852074.0,
|
||
|
|
"reward": 1.0786157131195069,
|
||
|
|
"reward_std": 0.10490404218435287,
|
||
|
|
"rewards/accuracy_reward": 0.39140625,
|
||
|
|
"rewards/brier_reward": 0.7659255743026734,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.36102074980735777,
|
||
|
|
"rewards/smce": -0.03446613140404224,
|
||
|
|
"step": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 640.0,
|
||
|
|
"completions/max_terminated_length": 400.8,
|
||
|
|
"completions/mean_length": 133.86708984375,
|
||
|
|
"completions/mean_terminated_length": 133.73010864257813,
|
||
|
|
"completions/min_length": 60.4,
|
||
|
|
"completions/min_terminated_length": 60.4,
|
||
|
|
"epoch": 0.288,
|
||
|
|
"grad_norm": 0.0008401651284657419,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 303248601.0,
|
||
|
|
"reward": 1.0780799865722657,
|
||
|
|
"reward_std": 0.09450059086084366,
|
||
|
|
"rewards/accuracy_reward": 0.38203125,
|
||
|
|
"rewards/brier_reward": 0.7744266152381897,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00078125,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.3482990264892578,
|
||
|
|
"rewards/smce": -0.05649419128894806,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 575.4,
|
||
|
|
"completions/max_terminated_length": 351.2,
|
||
|
|
"completions/mean_length": 136.93134765625,
|
||
|
|
"completions/mean_terminated_length": 136.79485168457032,
|
||
|
|
"completions/min_length": 63.6,
|
||
|
|
"completions/min_terminated_length": 63.6,
|
||
|
|
"epoch": 0.304,
|
||
|
|
"grad_norm": 0.0008501969859935343,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0001,
|
||
|
|
"num_tokens": 319864954.0,
|
||
|
|
"reward": 1.0633502006530762,
|
||
|
|
"reward_std": 0.09683705121278763,
|
||
|
|
"rewards/accuracy_reward": 0.3466796875,
|
||
|
|
"rewards/brier_reward": 0.7802187681198121,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0009765625,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.3442237675189972,
|
||
|
|
"rewards/smce": -0.03479156941175461,
|
||
|
|
"step": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 833.4,
|
||
|
|
"completions/max_terminated_length": 349.0,
|
||
|
|
"completions/mean_length": 139.363671875,
|
||
|
|
"completions/mean_terminated_length": 139.09087829589845,
|
||
|
|
"completions/min_length": 64.4,
|
||
|
|
"completions/min_terminated_length": 64.4,
|
||
|
|
"epoch": 0.32,
|
||
|
|
"grad_norm": 0.0007301330333575606,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 336301126.0,
|
||
|
|
"reward": 1.085144281387329,
|
||
|
|
"reward_std": 0.09936772584915161,
|
||
|
|
"rewards/accuracy_reward": 0.4005859375,
|
||
|
|
"rewards/brier_reward": 0.7700971603393555,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.357449609041214,
|
||
|
|
"rewards/smce": -0.04601215664297342,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.32,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 289.0,
|
||
|
|
"eval_completions/max_terminated_length": 289.0,
|
||
|
|
"eval_completions/mean_length": 139.2973175048828,
|
||
|
|
"eval_completions/mean_terminated_length": 139.2973175048828,
|
||
|
|
"eval_completions/min_length": 68.5,
|
||
|
|
"eval_completions/min_terminated_length": 68.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 336301126.0,
|
||
|
|
"eval_reward": 1.0518184304237366,
|
||
|
|
"eval_reward_std": 0.18363308906555176,
|
||
|
|
"eval_rewards/accuracy_reward": 0.318359375,
|
||
|
|
"eval_rewards/brier_reward": 0.785280853509903,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.001953125,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.3572597801685333,
|
||
|
|
"eval_rewards/smce": -0.04075256921350956,
|
||
|
|
"eval_runtime": 10.943,
|
||
|
|
"eval_samples_per_second": 45.691,
|
||
|
|
"eval_steps_per_second": 0.183,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 569.8,
|
||
|
|
"completions/max_terminated_length": 327.0,
|
||
|
|
"completions/mean_length": 139.15380859375,
|
||
|
|
"completions/mean_terminated_length": 138.88138122558593,
|
||
|
|
"completions/min_length": 59.2,
|
||
|
|
"completions/min_terminated_length": 59.2,
|
||
|
|
"epoch": 0.336,
|
||
|
|
"grad_norm": 0.0007222591666504741,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 352403181.0,
|
||
|
|
"reward": 1.075313401222229,
|
||
|
|
"reward_std": 0.1002796545624733,
|
||
|
|
"rewards/accuracy_reward": 0.3818359375,
|
||
|
|
"rewards/brier_reward": 0.7690893411636353,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.3662374377250671,
|
||
|
|
"rewards/smce": -0.06171097904443741,
|
||
|
|
"step": 105
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.000390625,
|
||
|
|
"completions/max_length": 1306.4,
|
||
|
|
"completions/max_terminated_length": 378.6,
|
||
|
|
"completions/mean_length": 139.9060546875,
|
||
|
|
"completions/mean_terminated_length": 139.3603302001953,
|
||
|
|
"completions/min_length": 62.6,
|
||
|
|
"completions/min_terminated_length": 62.6,
|
||
|
|
"epoch": 0.352,
|
||
|
|
"grad_norm": 0.0007559886435046792,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 368906091.0,
|
||
|
|
"reward": 1.089357614517212,
|
||
|
|
"reward_std": 0.09549021869897842,
|
||
|
|
"rewards/accuracy_reward": 0.405859375,
|
||
|
|
"rewards/brier_reward": 0.7732495069503784,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.3857435047626495,
|
||
|
|
"rewards/smce": -0.03754127398133278,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 619.4,
|
||
|
|
"completions/max_terminated_length": 407.4,
|
||
|
|
"completions/mean_length": 141.80400390625,
|
||
|
|
"completions/mean_terminated_length": 141.66767272949218,
|
||
|
|
"completions/min_length": 63.4,
|
||
|
|
"completions/min_terminated_length": 63.4,
|
||
|
|
"epoch": 0.368,
|
||
|
|
"grad_norm": 0.000884483742993325,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 385270324.0,
|
||
|
|
"reward": 1.0861406326293945,
|
||
|
|
"reward_std": 0.10510718375444413,
|
||
|
|
"rewards/accuracy_reward": 0.39833984375,
|
||
|
|
"rewards/brier_reward": 0.774041223526001,
|
||
|
|
"rewards/confidence_one_or_zero": 0.00048828125,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.39929391741752623,
|
||
|
|
"rewards/smce": -0.028393099457025527,
|
||
|
|
"step": 115
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1074.6,
|
||
|
|
"completions/max_terminated_length": 350.8,
|
||
|
|
"completions/mean_length": 145.78603515625,
|
||
|
|
"completions/mean_terminated_length": 145.37864685058594,
|
||
|
|
"completions/min_length": 68.6,
|
||
|
|
"completions/min_terminated_length": 68.6,
|
||
|
|
"epoch": 0.384,
|
||
|
|
"grad_norm": 0.0008524227887392044,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 401993605.0,
|
||
|
|
"reward": 1.0776931762695312,
|
||
|
|
"reward_std": 0.11128689795732498,
|
||
|
|
"rewards/accuracy_reward": 0.3865234375,
|
||
|
|
"rewards/brier_reward": 0.7691583395004272,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.4166277706623077,
|
||
|
|
"rewards/smce": -0.03233633115887642,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 580.6,
|
||
|
|
"completions/max_terminated_length": 349.2,
|
||
|
|
"completions/mean_length": 146.4333984375,
|
||
|
|
"completions/mean_terminated_length": 146.29757995605468,
|
||
|
|
"completions/min_length": 69.0,
|
||
|
|
"completions/min_terminated_length": 69.0,
|
||
|
|
"epoch": 0.4,
|
||
|
|
"grad_norm": 0.0008206322672776878,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": -0.0003,
|
||
|
|
"num_tokens": 418595739.0,
|
||
|
|
"reward": 1.1109728336334228,
|
||
|
|
"reward_std": 0.10586166381835938,
|
||
|
|
"rewards/accuracy_reward": 0.45146484375,
|
||
|
|
"rewards/brier_reward": 0.7708743572235107,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.999609375,
|
||
|
|
"rewards/mean_confidence_reward": 0.43074482679367065,
|
||
|
|
"rewards/smce": -0.037405839934945104,
|
||
|
|
"step": 125
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 579.6,
|
||
|
|
"completions/max_terminated_length": 356.8,
|
||
|
|
"completions/mean_length": 150.88515625,
|
||
|
|
"completions/mean_terminated_length": 150.7495086669922,
|
||
|
|
"completions/min_length": 66.4,
|
||
|
|
"completions/min_terminated_length": 66.4,
|
||
|
|
"epoch": 0.416,
|
||
|
|
"grad_norm": 0.0009291012538596988,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 435183651.0,
|
||
|
|
"reward": 1.106221318244934,
|
||
|
|
"reward_std": 0.11768909841775894,
|
||
|
|
"rewards/accuracy_reward": 0.44599609375,
|
||
|
|
"rewards/brier_reward": 0.7666454672813415,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4515750050544739,
|
||
|
|
"rewards/smce": -0.04493560083210468,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 864.8,
|
||
|
|
"completions/max_terminated_length": 389.6,
|
||
|
|
"completions/mean_length": 156.02626953125,
|
||
|
|
"completions/mean_terminated_length": 155.7564697265625,
|
||
|
|
"completions/min_length": 68.4,
|
||
|
|
"completions/min_terminated_length": 68.4,
|
||
|
|
"epoch": 0.432,
|
||
|
|
"grad_norm": 0.000898000958841294,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 451773584.0,
|
||
|
|
"reward": 1.106620478630066,
|
||
|
|
"reward_std": 0.11107992678880692,
|
||
|
|
"rewards/accuracy_reward": 0.4431640625,
|
||
|
|
"rewards/brier_reward": 0.7702758431434631,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0001953125,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.44765183329582214,
|
||
|
|
"rewards/smce": -0.044996288046240805,
|
||
|
|
"step": 135
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 600.0,
|
||
|
|
"completions/max_terminated_length": 399.6,
|
||
|
|
"completions/mean_length": 157.69306640625,
|
||
|
|
"completions/mean_terminated_length": 157.5587921142578,
|
||
|
|
"completions/min_length": 69.0,
|
||
|
|
"completions/min_terminated_length": 69.0,
|
||
|
|
"epoch": 0.448,
|
||
|
|
"grad_norm": 0.0008736915187910199,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 468267273.0,
|
||
|
|
"reward": 1.096074390411377,
|
||
|
|
"reward_std": 0.11458188891410828,
|
||
|
|
"rewards/accuracy_reward": 0.41748046875,
|
||
|
|
"rewards/brier_reward": 0.7748677492141723,
|
||
|
|
"rewards/confidence_one_or_zero": 0.000390625,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.44305351972579954,
|
||
|
|
"rewards/smce": -0.04980777129530907,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 378.2,
|
||
|
|
"completions/max_terminated_length": 378.2,
|
||
|
|
"completions/mean_length": 157.0857421875,
|
||
|
|
"completions/mean_terminated_length": 157.0857421875,
|
||
|
|
"completions/min_length": 68.4,
|
||
|
|
"completions/min_terminated_length": 68.4,
|
||
|
|
"epoch": 0.464,
|
||
|
|
"grad_norm": 0.0008124883752316236,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 484963223.0,
|
||
|
|
"reward": 1.1068456649780274,
|
||
|
|
"reward_std": 0.11299627423286437,
|
||
|
|
"rewards/accuracy_reward": 0.434375,
|
||
|
|
"rewards/brier_reward": 0.7793171048164368,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4311175882816315,
|
||
|
|
"rewards/smce": -0.01590832956135273,
|
||
|
|
"step": 145
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 583.2,
|
||
|
|
"completions/max_terminated_length": 345.8,
|
||
|
|
"completions/mean_length": 157.90966796875,
|
||
|
|
"completions/mean_terminated_length": 157.7749816894531,
|
||
|
|
"completions/min_length": 68.6,
|
||
|
|
"completions/min_terminated_length": 68.6,
|
||
|
|
"epoch": 0.48,
|
||
|
|
"grad_norm": 0.0008954492514021695,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 501711034.0,
|
||
|
|
"reward": 1.0894078254699706,
|
||
|
|
"reward_std": 0.10769533514976501,
|
||
|
|
"rewards/accuracy_reward": 0.39990234375,
|
||
|
|
"rewards/brier_reward": 0.7791118502616883,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.43644275069236754,
|
||
|
|
"rewards/smce": -0.040997632779181004,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.48,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 290.0,
|
||
|
|
"eval_completions/max_terminated_length": 290.0,
|
||
|
|
"eval_completions/mean_length": 157.6277847290039,
|
||
|
|
"eval_completions/mean_terminated_length": 157.6277847290039,
|
||
|
|
"eval_completions/min_length": 85.0,
|
||
|
|
"eval_completions/min_terminated_length": 85.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 501711034.0,
|
||
|
|
"eval_reward": 1.0702556371688843,
|
||
|
|
"eval_reward_std": 0.2216203734278679,
|
||
|
|
"eval_rewards/accuracy_reward": 0.35546875,
|
||
|
|
"eval_rewards/brier_reward": 0.7850488126277924,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.4248046875,
|
||
|
|
"eval_rewards/smce": -0.07097656466066837,
|
||
|
|
"eval_runtime": 10.9948,
|
||
|
|
"eval_samples_per_second": 45.476,
|
||
|
|
"eval_steps_per_second": 0.182,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1074.8,
|
||
|
|
"completions/max_terminated_length": 377.8,
|
||
|
|
"completions/mean_length": 158.95439453125,
|
||
|
|
"completions/mean_terminated_length": 158.55074157714844,
|
||
|
|
"completions/min_length": 69.0,
|
||
|
|
"completions/min_terminated_length": 69.0,
|
||
|
|
"epoch": 0.496,
|
||
|
|
"grad_norm": 0.000837022322230041,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 518148647.0,
|
||
|
|
"reward": 1.1130841255187989,
|
||
|
|
"reward_std": 0.10837481170892715,
|
||
|
|
"rewards/accuracy_reward": 0.45146484375,
|
||
|
|
"rewards/brier_reward": 0.7749992847442627,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0005859375,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.439496785402298,
|
||
|
|
"rewards/smce": -0.03746632831171155,
|
||
|
|
"step": 155
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 371.0,
|
||
|
|
"completions/max_terminated_length": 371.0,
|
||
|
|
"completions/mean_length": 161.6998046875,
|
||
|
|
"completions/mean_terminated_length": 161.6998046875,
|
||
|
|
"completions/min_length": 67.6,
|
||
|
|
"completions/min_terminated_length": 67.6,
|
||
|
|
"epoch": 0.512,
|
||
|
|
"grad_norm": 0.0007171882316470146,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0,
|
||
|
|
"num_tokens": 534711397.0,
|
||
|
|
"reward": 1.0941582918167114,
|
||
|
|
"reward_std": 0.09756620526313782,
|
||
|
|
"rewards/accuracy_reward": 0.40478515625,
|
||
|
|
"rewards/brier_reward": 0.7835341572761536,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4164893627166748,
|
||
|
|
"rewards/smce": -0.03459956496953964,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 615.2,
|
||
|
|
"completions/max_terminated_length": 392.8,
|
||
|
|
"completions/mean_length": 165.54208984375,
|
||
|
|
"completions/mean_terminated_length": 165.4082794189453,
|
||
|
|
"completions/min_length": 75.2,
|
||
|
|
"completions/min_terminated_length": 75.2,
|
||
|
|
"epoch": 0.528,
|
||
|
|
"grad_norm": 0.0009193150326609612,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 551478228.0,
|
||
|
|
"reward": 1.1048997163772583,
|
||
|
|
"reward_std": 0.10788022428750992,
|
||
|
|
"rewards/accuracy_reward": 0.42314453125,
|
||
|
|
"rewards/brier_reward": 0.7867572665214538,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4206361770629883,
|
||
|
|
"rewards/smce": -0.05594331808388233,
|
||
|
|
"step": 165
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1069.4,
|
||
|
|
"completions/max_terminated_length": 372.4,
|
||
|
|
"completions/mean_length": 164.344140625,
|
||
|
|
"completions/mean_terminated_length": 163.94191284179686,
|
||
|
|
"completions/min_length": 79.0,
|
||
|
|
"completions/min_terminated_length": 79.0,
|
||
|
|
"epoch": 0.544,
|
||
|
|
"grad_norm": 0.0009100640309043229,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 568289304.0,
|
||
|
|
"reward": 1.1251786947250366,
|
||
|
|
"reward_std": 0.1073757141828537,
|
||
|
|
"rewards/accuracy_reward": 0.4765625,
|
||
|
|
"rewards/brier_reward": 0.7740918874740601,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.44392098784446715,
|
||
|
|
"rewards/smce": -0.04849136099219322,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 864.6,
|
||
|
|
"completions/max_terminated_length": 431.4,
|
||
|
|
"completions/mean_length": 167.23515625,
|
||
|
|
"completions/mean_terminated_length": 166.96788024902344,
|
||
|
|
"completions/min_length": 79.6,
|
||
|
|
"completions/min_terminated_length": 79.6,
|
||
|
|
"epoch": 0.56,
|
||
|
|
"grad_norm": 0.0008807600825093687,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 585010496.0,
|
||
|
|
"reward": 1.1237701177597046,
|
||
|
|
"reward_std": 0.10904805213212967,
|
||
|
|
"rewards/accuracy_reward": 0.465234375,
|
||
|
|
"rewards/brier_reward": 0.7825065374374389,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4573706090450287,
|
||
|
|
"rewards/smce": -0.06266507729887963,
|
||
|
|
"step": 175
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 841.8,
|
||
|
|
"completions/max_terminated_length": 410.6,
|
||
|
|
"completions/mean_length": 171.80546875,
|
||
|
|
"completions/mean_terminated_length": 171.5396240234375,
|
||
|
|
"completions/min_length": 84.2,
|
||
|
|
"completions/min_terminated_length": 84.2,
|
||
|
|
"epoch": 0.576,
|
||
|
|
"grad_norm": 0.000868572446051985,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 601983000.0,
|
||
|
|
"reward": 1.120293092727661,
|
||
|
|
"reward_std": 0.10446289032697678,
|
||
|
|
"rewards/accuracy_reward": 0.44287109375,
|
||
|
|
"rewards/brier_reward": 0.7979134917259216,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4504714846611023,
|
||
|
|
"rewards/smce": -0.03961756564676762,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 719.2,
|
||
|
|
"completions/max_terminated_length": 499.2,
|
||
|
|
"completions/mean_length": 178.27021484375,
|
||
|
|
"completions/mean_terminated_length": 178.13722534179686,
|
||
|
|
"completions/min_length": 89.2,
|
||
|
|
"completions/min_terminated_length": 89.2,
|
||
|
|
"epoch": 0.592,
|
||
|
|
"grad_norm": 0.0007287701591849327,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 618830951.0,
|
||
|
|
"reward": 1.0865333557128907,
|
||
|
|
"reward_std": 0.1005751445889473,
|
||
|
|
"rewards/accuracy_reward": 0.38828125,
|
||
|
|
"rewards/brier_reward": 0.7848896384239197,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4494062900543213,
|
||
|
|
"rewards/smce": -0.07449966222047806,
|
||
|
|
"step": 185
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 643.8,
|
||
|
|
"completions/max_terminated_length": 412.2,
|
||
|
|
"completions/mean_length": 180.780859375,
|
||
|
|
"completions/mean_terminated_length": 180.64830932617187,
|
||
|
|
"completions/min_length": 86.8,
|
||
|
|
"completions/min_terminated_length": 86.8,
|
||
|
|
"epoch": 0.608,
|
||
|
|
"grad_norm": 0.000843865389470011,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 635613475.0,
|
||
|
|
"reward": 1.1116358280181884,
|
||
|
|
"reward_std": 0.0987422376871109,
|
||
|
|
"rewards/accuracy_reward": 0.43408203125,
|
||
|
|
"rewards/brier_reward": 0.7892889738082886,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.43660807609558105,
|
||
|
|
"rewards/smce": -0.025845246762037276,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 946.4,
|
||
|
|
"completions/max_terminated_length": 506.8,
|
||
|
|
"completions/mean_length": 180.91884765625,
|
||
|
|
"completions/mean_terminated_length": 180.65436096191405,
|
||
|
|
"completions/min_length": 88.4,
|
||
|
|
"completions/min_terminated_length": 88.4,
|
||
|
|
"epoch": 0.624,
|
||
|
|
"grad_norm": 0.0007394336280412972,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 652432676.0,
|
||
|
|
"reward": 1.1127588748931885,
|
||
|
|
"reward_std": 0.09222938418388367,
|
||
|
|
"rewards/accuracy_reward": 0.4365234375,
|
||
|
|
"rewards/brier_reward": 0.7891914367675781,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.41735615730285647,
|
||
|
|
"rewards/smce": -0.02638731971383095,
|
||
|
|
"step": 195
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 706.4,
|
||
|
|
"completions/max_terminated_length": 476.2,
|
||
|
|
"completions/mean_length": 179.44296875,
|
||
|
|
"completions/mean_terminated_length": 179.31036071777345,
|
||
|
|
"completions/min_length": 83.0,
|
||
|
|
"completions/min_terminated_length": 83.0,
|
||
|
|
"epoch": 0.64,
|
||
|
|
"grad_norm": 0.0009229676797986031,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 669289564.0,
|
||
|
|
"reward": 1.1244325637817383,
|
||
|
|
"reward_std": 0.10907497704029083,
|
||
|
|
"rewards/accuracy_reward": 0.463671875,
|
||
|
|
"rewards/brier_reward": 0.7852943778038025,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4432168066501617,
|
||
|
|
"rewards/smce": -0.04280035831034183,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.64,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 344.5,
|
||
|
|
"eval_completions/max_terminated_length": 344.5,
|
||
|
|
"eval_completions/mean_length": 178.71320343017578,
|
||
|
|
"eval_completions/mean_terminated_length": 178.71320343017578,
|
||
|
|
"eval_completions/min_length": 103.0,
|
||
|
|
"eval_completions/min_terminated_length": 103.0,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 669289564.0,
|
||
|
|
"eval_reward": 1.074850082397461,
|
||
|
|
"eval_reward_std": 0.24065522849559784,
|
||
|
|
"eval_rewards/accuracy_reward": 0.359375,
|
||
|
|
"eval_rewards/brier_reward": 0.7903341948986053,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.45826172828674316,
|
||
|
|
"eval_rewards/smce": -0.09919121861457825,
|
||
|
|
"eval_runtime": 11.9627,
|
||
|
|
"eval_samples_per_second": 41.797,
|
||
|
|
"eval_steps_per_second": 0.167,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 665.4,
|
||
|
|
"completions/max_terminated_length": 434.0,
|
||
|
|
"completions/mean_length": 175.375,
|
||
|
|
"completions/mean_terminated_length": 175.24189147949218,
|
||
|
|
"completions/min_length": 82.6,
|
||
|
|
"completions/min_terminated_length": 82.6,
|
||
|
|
"epoch": 0.656,
|
||
|
|
"grad_norm": 0.0008042529225349426,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"num_tokens": 686066204.0,
|
||
|
|
"reward": 1.11544828414917,
|
||
|
|
"reward_std": 0.11178317070007324,
|
||
|
|
"rewards/accuracy_reward": 0.448046875,
|
||
|
|
"rewards/brier_reward": 0.7829502701759339,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4799645602703094,
|
||
|
|
"rewards/smce": -0.03865515626966953,
|
||
|
|
"step": 205
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 648.8,
|
||
|
|
"completions/max_terminated_length": 424.0,
|
||
|
|
"completions/mean_length": 172.976953125,
|
||
|
|
"completions/mean_terminated_length": 172.84388122558593,
|
||
|
|
"completions/min_length": 82.8,
|
||
|
|
"completions/min_terminated_length": 82.8,
|
||
|
|
"epoch": 0.672,
|
||
|
|
"grad_norm": 0.000913208001293242,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 702852176.0,
|
||
|
|
"reward": 1.1459183931350707,
|
||
|
|
"reward_std": 0.11151646673679352,
|
||
|
|
"rewards/accuracy_reward": 0.50302734375,
|
||
|
|
"rewards/brier_reward": 0.7889090418815613,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4935117304325104,
|
||
|
|
"rewards/smce": -0.02932633813470602,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 533.8,
|
||
|
|
"completions/max_terminated_length": 533.8,
|
||
|
|
"completions/mean_length": 169.51435546875,
|
||
|
|
"completions/mean_terminated_length": 169.51435546875,
|
||
|
|
"completions/min_length": 83.4,
|
||
|
|
"completions/min_terminated_length": 83.4,
|
||
|
|
"epoch": 0.688,
|
||
|
|
"grad_norm": 0.0009185533272102475,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 719669411.0,
|
||
|
|
"reward": 1.11937415599823,
|
||
|
|
"reward_std": 0.1056325763463974,
|
||
|
|
"rewards/accuracy_reward": 0.445703125,
|
||
|
|
"rewards/brier_reward": 0.7930505633354187,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4931591808795929,
|
||
|
|
"rewards/smce": -0.0630765751004219,
|
||
|
|
"step": 215
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 830.6,
|
||
|
|
"completions/max_terminated_length": 371.4,
|
||
|
|
"completions/mean_length": 170.2216796875,
|
||
|
|
"completions/mean_terminated_length": 169.9547332763672,
|
||
|
|
"completions/min_length": 84.6,
|
||
|
|
"completions/min_terminated_length": 84.6,
|
||
|
|
"epoch": 0.704,
|
||
|
|
"grad_norm": 0.0010203915880993009,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 736362849.0,
|
||
|
|
"reward": 1.1581217765808105,
|
||
|
|
"reward_std": 0.11780242323875427,
|
||
|
|
"rewards/accuracy_reward": 0.53212890625,
|
||
|
|
"rewards/brier_reward": 0.78431156873703,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5082412004470825,
|
||
|
|
"rewards/smce": -0.026287396252155305,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 669.2,
|
||
|
|
"completions/max_terminated_length": 448.0,
|
||
|
|
"completions/mean_length": 173.9283203125,
|
||
|
|
"completions/mean_terminated_length": 173.79610290527344,
|
||
|
|
"completions/min_length": 87.6,
|
||
|
|
"completions/min_terminated_length": 87.6,
|
||
|
|
"epoch": 0.72,
|
||
|
|
"grad_norm": 0.0008801869698800147,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0003,
|
||
|
|
"num_tokens": 753279939.0,
|
||
|
|
"reward": 1.1250783205032349,
|
||
|
|
"reward_std": 0.11430736482143403,
|
||
|
|
"rewards/accuracy_reward": 0.4568359375,
|
||
|
|
"rewards/brier_reward": 0.7934238076210022,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.49130371809005735,
|
||
|
|
"rewards/smce": -0.06444230675697327,
|
||
|
|
"step": 225
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 849.2,
|
||
|
|
"completions/max_terminated_length": 405.2,
|
||
|
|
"completions/mean_length": 174.85498046875,
|
||
|
|
"completions/mean_terminated_length": 174.58867492675782,
|
||
|
|
"completions/min_length": 82.8,
|
||
|
|
"completions/min_terminated_length": 82.8,
|
||
|
|
"epoch": 0.736,
|
||
|
|
"grad_norm": 0.0009612944559194148,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 770042742.0,
|
||
|
|
"reward": 1.1493725538253785,
|
||
|
|
"reward_std": 0.10344905704259873,
|
||
|
|
"rewards/accuracy_reward": 0.5033203125,
|
||
|
|
"rewards/brier_reward": 0.7956208944320678,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5045771479606629,
|
||
|
|
"rewards/smce": -0.018064015358686448,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 644.2,
|
||
|
|
"completions/max_terminated_length": 427.4,
|
||
|
|
"completions/mean_length": 179.42392578125,
|
||
|
|
"completions/mean_terminated_length": 179.29153442382812,
|
||
|
|
"completions/min_length": 87.8,
|
||
|
|
"completions/min_terminated_length": 87.8,
|
||
|
|
"epoch": 0.752,
|
||
|
|
"grad_norm": 0.0010464686201885343,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 786916075.0,
|
||
|
|
"reward": 1.1549260854721068,
|
||
|
|
"reward_std": 0.11921477615833283,
|
||
|
|
"rewards/accuracy_reward": 0.5169921875,
|
||
|
|
"rewards/brier_reward": 0.7929610848426819,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.5239619135856628,
|
||
|
|
"rewards/smce": -0.04414962157607079,
|
||
|
|
"step": 235
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 799.4,
|
||
|
|
"completions/max_terminated_length": 617.8,
|
||
|
|
"completions/mean_length": 184.51640625,
|
||
|
|
"completions/mean_terminated_length": 184.38465270996093,
|
||
|
|
"completions/min_length": 87.4,
|
||
|
|
"completions/min_terminated_length": 87.4,
|
||
|
|
"epoch": 0.768,
|
||
|
|
"grad_norm": 0.0009205514215864241,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 804057267.0,
|
||
|
|
"reward": 1.1374267578125,
|
||
|
|
"reward_std": 0.10838203877210617,
|
||
|
|
"rewards/accuracy_reward": 0.4748046875,
|
||
|
|
"rewards/brier_reward": 0.800150191783905,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.5018200159072876,
|
||
|
|
"rewards/smce": -0.04639698397368193,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 458.6,
|
||
|
|
"completions/max_terminated_length": 458.6,
|
||
|
|
"completions/mean_length": 183.49248046875,
|
||
|
|
"completions/mean_terminated_length": 183.49248046875,
|
||
|
|
"completions/min_length": 85.4,
|
||
|
|
"completions/min_terminated_length": 85.4,
|
||
|
|
"epoch": 0.784,
|
||
|
|
"grad_norm": 0.0008747904212214053,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 820903718.0,
|
||
|
|
"reward": 1.140303111076355,
|
||
|
|
"reward_std": 0.1141647920012474,
|
||
|
|
"rewards/accuracy_reward": 0.493359375,
|
||
|
|
"rewards/brier_reward": 0.7873458147048951,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.49045605063438413,
|
||
|
|
"rewards/smce": -0.02265973724424839,
|
||
|
|
"step": 245
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 917.6,
|
||
|
|
"completions/max_terminated_length": 471.4,
|
||
|
|
"completions/mean_length": 185.45810546875,
|
||
|
|
"completions/mean_terminated_length": 185.19440307617188,
|
||
|
|
"completions/min_length": 96.2,
|
||
|
|
"completions/min_terminated_length": 96.2,
|
||
|
|
"epoch": 0.8,
|
||
|
|
"grad_norm": 0.0008962493157014251,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 837924505.0,
|
||
|
|
"reward": 1.1438107013702392,
|
||
|
|
"reward_std": 0.10105291306972504,
|
||
|
|
"rewards/accuracy_reward": 0.488671875,
|
||
|
|
"rewards/brier_reward": 0.7992471814155578,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.47882911562919617,
|
||
|
|
"rewards/smce": -0.056229662522673605,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.8,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 354.5,
|
||
|
|
"eval_completions/max_terminated_length": 354.5,
|
||
|
|
"eval_completions/mean_length": 188.99618530273438,
|
||
|
|
"eval_completions/mean_terminated_length": 188.99618530273438,
|
||
|
|
"eval_completions/min_length": 108.5,
|
||
|
|
"eval_completions/min_terminated_length": 108.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 837924505.0,
|
||
|
|
"eval_reward": 1.0744569301605225,
|
||
|
|
"eval_reward_std": 0.2566024512052536,
|
||
|
|
"eval_rewards/accuracy_reward": 0.345703125,
|
||
|
|
"eval_rewards/brier_reward": 0.80322265625,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.47246094048023224,
|
||
|
|
"eval_rewards/smce": -0.12773926183581352,
|
||
|
|
"eval_runtime": 13.1459,
|
||
|
|
"eval_samples_per_second": 38.035,
|
||
|
|
"eval_steps_per_second": 0.152,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 463.6,
|
||
|
|
"completions/max_terminated_length": 463.6,
|
||
|
|
"completions/mean_length": 187.904296875,
|
||
|
|
"completions/mean_terminated_length": 187.904296875,
|
||
|
|
"completions/min_length": 94.8,
|
||
|
|
"completions/min_terminated_length": 94.8,
|
||
|
|
"epoch": 0.816,
|
||
|
|
"grad_norm": 0.0009017204283736646,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 855033413.0,
|
||
|
|
"reward": 1.1587502717971803,
|
||
|
|
"reward_std": 0.09949774891138077,
|
||
|
|
"rewards/accuracy_reward": 0.50849609375,
|
||
|
|
"rewards/brier_reward": 0.8090077996253967,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.4960273504257202,
|
||
|
|
"rewards/smce": -0.04275580998510122,
|
||
|
|
"step": 255
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 560.0,
|
||
|
|
"completions/max_terminated_length": 560.0,
|
||
|
|
"completions/mean_length": 194.63994140625,
|
||
|
|
"completions/mean_terminated_length": 194.63994140625,
|
||
|
|
"completions/min_length": 100.4,
|
||
|
|
"completions/min_terminated_length": 100.4,
|
||
|
|
"epoch": 0.832,
|
||
|
|
"grad_norm": 0.000874867953825742,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 872085822.0,
|
||
|
|
"reward": 1.1352928638458253,
|
||
|
|
"reward_std": 0.10674976855516434,
|
||
|
|
"rewards/accuracy_reward": 0.47353515625,
|
||
|
|
"rewards/brier_reward": 0.7970545411109924,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.505438482761383,
|
||
|
|
"rewards/smce": -0.04949844591319561,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 941.2,
|
||
|
|
"completions/max_terminated_length": 491.8,
|
||
|
|
"completions/mean_length": 198.4744140625,
|
||
|
|
"completions/mean_terminated_length": 198.21253662109376,
|
||
|
|
"completions/min_length": 105.2,
|
||
|
|
"completions/min_terminated_length": 105.2,
|
||
|
|
"epoch": 0.848,
|
||
|
|
"grad_norm": 0.0008845736738294363,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 889214136.0,
|
||
|
|
"reward": 1.1375447273254395,
|
||
|
|
"reward_std": 0.09591394364833832,
|
||
|
|
"rewards/accuracy_reward": 0.47041015625,
|
||
|
|
"rewards/brier_reward": 0.8048787593841553,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.5062236309051513,
|
||
|
|
"rewards/smce": -0.0516842994838953,
|
||
|
|
"step": 265
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 509.0,
|
||
|
|
"completions/max_terminated_length": 509.0,
|
||
|
|
"completions/mean_length": 199.49775390625,
|
||
|
|
"completions/mean_terminated_length": 199.49775390625,
|
||
|
|
"completions/min_length": 97.4,
|
||
|
|
"completions/min_terminated_length": 97.4,
|
||
|
|
"epoch": 0.864,
|
||
|
|
"grad_norm": 0.0005732330610044301,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 906454049.0,
|
||
|
|
"reward": 1.1363605737686158,
|
||
|
|
"reward_std": 0.09182919263839721,
|
||
|
|
"rewards/accuracy_reward": 0.4693359375,
|
||
|
|
"rewards/brier_reward": 0.8035852432250976,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4373330116271973,
|
||
|
|
"rewards/smce": -0.05709770061075688,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 683.2,
|
||
|
|
"completions/max_terminated_length": 484.0,
|
||
|
|
"completions/mean_length": 206.20185546875,
|
||
|
|
"completions/mean_terminated_length": 206.07252502441406,
|
||
|
|
"completions/min_length": 105.6,
|
||
|
|
"completions/min_terminated_length": 105.6,
|
||
|
|
"epoch": 0.88,
|
||
|
|
"grad_norm": 0.0007916499162092805,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 923661428.0,
|
||
|
|
"reward": 1.128675103187561,
|
||
|
|
"reward_std": 0.09593857079744339,
|
||
|
|
"rewards/accuracy_reward": 0.47060546875,
|
||
|
|
"rewards/brier_reward": 0.7869455933570861,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4107128918170929,
|
||
|
|
"rewards/smce": -0.06385259106755256,
|
||
|
|
"step": 275
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 714.8,
|
||
|
|
"completions/max_terminated_length": 508.6,
|
||
|
|
"completions/mean_length": 206.65712890625,
|
||
|
|
"completions/mean_terminated_length": 206.5268341064453,
|
||
|
|
"completions/min_length": 105.6,
|
||
|
|
"completions/min_terminated_length": 105.6,
|
||
|
|
"epoch": 0.896,
|
||
|
|
"grad_norm": 0.0008560984279029071,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 940891261.0,
|
||
|
|
"reward": 1.1234555006027223,
|
||
|
|
"reward_std": 0.10874682813882827,
|
||
|
|
"rewards/accuracy_reward": 0.45595703125,
|
||
|
|
"rewards/brier_reward": 0.7910539627075195,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4712998032569885,
|
||
|
|
"rewards/smce": -0.033051177859306335,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 761.2,
|
||
|
|
"completions/max_terminated_length": 551.4,
|
||
|
|
"completions/mean_length": 200.74912109375,
|
||
|
|
"completions/mean_terminated_length": 200.61900634765624,
|
||
|
|
"completions/min_length": 100.4,
|
||
|
|
"completions/min_terminated_length": 100.4,
|
||
|
|
"epoch": 0.912,
|
||
|
|
"grad_norm": 0.0010159144876524806,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0007,
|
||
|
|
"num_tokens": 957992660.0,
|
||
|
|
"reward": 1.1473841905593871,
|
||
|
|
"reward_std": 0.10081676095724106,
|
||
|
|
"rewards/accuracy_reward": 0.49599609375,
|
||
|
|
"rewards/brier_reward": 0.7989710807800293,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.4727851629257202,
|
||
|
|
"rewards/smce": -0.04406613744795322,
|
||
|
|
"step": 285
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0001953125,
|
||
|
|
"completions/max_length": 691.6,
|
||
|
|
"completions/max_terminated_length": 482.4,
|
||
|
|
"completions/mean_length": 191.688671875,
|
||
|
|
"completions/mean_terminated_length": 191.4256622314453,
|
||
|
|
"completions/min_length": 96.4,
|
||
|
|
"completions/min_terminated_length": 96.4,
|
||
|
|
"epoch": 0.928,
|
||
|
|
"grad_norm": 0.0008976598037406802,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 974623840.0,
|
||
|
|
"reward": 1.1475050926208497,
|
||
|
|
"reward_std": 0.0983053594827652,
|
||
|
|
"rewards/accuracy_reward": 0.50693359375,
|
||
|
|
"rewards/brier_reward": 0.7882754921913147,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.9998046875,
|
||
|
|
"rewards/mean_confidence_reward": 0.48968067169189455,
|
||
|
|
"rewards/smce": -0.04574570134282112,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 695.0,
|
||
|
|
"completions/max_terminated_length": 478.4,
|
||
|
|
"completions/mean_length": 191.94453125,
|
||
|
|
"completions/mean_terminated_length": 191.81333618164064,
|
||
|
|
"completions/min_length": 98.2,
|
||
|
|
"completions/min_terminated_length": 98.2,
|
||
|
|
"epoch": 0.944,
|
||
|
|
"grad_norm": 0.0008249140810221434,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 991602376.0,
|
||
|
|
"reward": 1.1368897676467895,
|
||
|
|
"reward_std": 0.09058477282524109,
|
||
|
|
"rewards/accuracy_reward": 0.4677734375,
|
||
|
|
"rewards/brier_reward": 0.8061062097549438,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4747841954231262,
|
||
|
|
"rewards/smce": -0.03453637734055519,
|
||
|
|
"step": 295
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 9.765625e-05,
|
||
|
|
"completions/max_length": 680.2,
|
||
|
|
"completions/max_terminated_length": 487.2,
|
||
|
|
"completions/mean_length": 192.143359375,
|
||
|
|
"completions/mean_terminated_length": 192.01217956542968,
|
||
|
|
"completions/min_length": 92.4,
|
||
|
|
"completions/min_terminated_length": 92.4,
|
||
|
|
"epoch": 0.96,
|
||
|
|
"grad_norm": 0.0008051492040976882,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0006,
|
||
|
|
"num_tokens": 1008719364.0,
|
||
|
|
"reward": 1.1513455629348754,
|
||
|
|
"reward_std": 0.0891915887594223,
|
||
|
|
"rewards/accuracy_reward": 0.49482421875,
|
||
|
|
"rewards/brier_reward": 0.8079674720764161,
|
||
|
|
"rewards/confidence_one_or_zero": 9.765625e-05,
|
||
|
|
"rewards/format_reward": 0.99990234375,
|
||
|
|
"rewards/mean_confidence_reward": 0.4651210904121399,
|
||
|
|
"rewards/smce": -0.03700222373008728,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.96,
|
||
|
|
"eval_completions/clipped_ratio": 0.0,
|
||
|
|
"eval_completions/max_length": 372.0,
|
||
|
|
"eval_completions/max_terminated_length": 372.0,
|
||
|
|
"eval_completions/mean_length": 192.57955932617188,
|
||
|
|
"eval_completions/mean_terminated_length": 192.57955932617188,
|
||
|
|
"eval_completions/min_length": 103.5,
|
||
|
|
"eval_completions/min_terminated_length": 103.5,
|
||
|
|
"eval_loss": 0.0,
|
||
|
|
"eval_num_tokens": 1008719364.0,
|
||
|
|
"eval_reward": 1.0884253978729248,
|
||
|
|
"eval_reward_std": 0.2481953427195549,
|
||
|
|
"eval_rewards/accuracy_reward": 0.369140625,
|
||
|
|
"eval_rewards/brier_reward": 0.8077171742916107,
|
||
|
|
"eval_rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"eval_rewards/format_reward": 1.0,
|
||
|
|
"eval_rewards/mean_confidence_reward": 0.44535157084465027,
|
||
|
|
"eval_rewards/smce": -0.07722852565348148,
|
||
|
|
"eval_runtime": 12.6348,
|
||
|
|
"eval_samples_per_second": 39.573,
|
||
|
|
"eval_steps_per_second": 0.158,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00029296875,
|
||
|
|
"completions/max_length": 1102.6,
|
||
|
|
"completions/max_terminated_length": 473.4,
|
||
|
|
"completions/mean_length": 192.79931640625,
|
||
|
|
"completions/mean_terminated_length": 192.40595397949218,
|
||
|
|
"completions/min_length": 102.4,
|
||
|
|
"completions/min_terminated_length": 102.4,
|
||
|
|
"epoch": 0.976,
|
||
|
|
"grad_norm": 0.0008862409740686417,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"num_tokens": 1025823645.0,
|
||
|
|
"reward": 1.1745065450668335,
|
||
|
|
"reward_std": 0.08961422890424728,
|
||
|
|
"rewards/accuracy_reward": 0.53203125,
|
||
|
|
"rewards/brier_reward": 0.817278528213501,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99970703125,
|
||
|
|
"rewards/mean_confidence_reward": 0.48915820717811587,
|
||
|
|
"rewards/smce": -0.04640830308198929,
|
||
|
|
"step": 305
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 483.8,
|
||
|
|
"completions/max_terminated_length": 483.8,
|
||
|
|
"completions/mean_length": 196.373828125,
|
||
|
|
"completions/mean_terminated_length": 196.373828125,
|
||
|
|
"completions/min_length": 104.2,
|
||
|
|
"completions/min_terminated_length": 104.2,
|
||
|
|
"epoch": 0.992,
|
||
|
|
"grad_norm": 0.0009224271052516997,
|
||
|
|
"learning_rate": 1e-06,
|
||
|
|
"loss": 0.0002,
|
||
|
|
"num_tokens": 1042824881.0,
|
||
|
|
"reward": 1.1565107822418212,
|
||
|
|
"reward_std": 0.08400667905807495,
|
||
|
|
"rewards/accuracy_reward": 0.50283203125,
|
||
|
|
"rewards/brier_reward": 0.8101913809776307,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 1.0,
|
||
|
|
"rewards/mean_confidence_reward": 0.5155996084213257,
|
||
|
|
"rewards/smce": -0.02968781553208828,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.00048828125,
|
||
|
|
"completions/max_length": 1057.5,
|
||
|
|
"completions/max_terminated_length": 501.5,
|
||
|
|
"completions/mean_length": 202.44482421875,
|
||
|
|
"completions/mean_terminated_length": 201.7928466796875,
|
||
|
|
"completions/min_length": 97.0,
|
||
|
|
"completions/min_terminated_length": 97.0,
|
||
|
|
"epoch": 0.9984,
|
||
|
|
"num_tokens": 1049742511.0,
|
||
|
|
"reward": 1.1453857421875,
|
||
|
|
"reward_std": 0.09386108070611954,
|
||
|
|
"rewards/accuracy_reward": 0.48779296875,
|
||
|
|
"rewards/brier_reward": 0.8034725785255432,
|
||
|
|
"rewards/confidence_one_or_zero": 0.0,
|
||
|
|
"rewards/format_reward": 0.99951171875,
|
||
|
|
"rewards/mean_confidence_reward": 0.49439698457717896,
|
||
|
|
"rewards/smce": -0.06732712313532829,
|
||
|
|
"step": 312,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.00036888672133127094,
|
||
|
|
"train_runtime": 33617.5864,
|
||
|
|
"train_samples_per_second": 0.595,
|
||
|
|
"train_steps_per_second": 0.009
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 5,
|
||
|
|
"max_steps": 312,
|
||
|
|
"num_input_tokens_seen": 1049742511,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 60,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 4,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|