7694 lines
336 KiB
JSON
7694 lines
336 KiB
JSON
|
{
|
||
|
|
"best_global_step": null,
|
||
|
|
"best_metric": null,
|
||
|
|
"best_model_checkpoint": null,
|
||
|
|
"epoch": 0.12002133712660028,
|
||
|
|
"eval_steps": 500,
|
||
|
|
"global_step": 450,
|
||
|
|
"is_hyper_param_search": false,
|
||
|
|
"is_local_process_zero": true,
|
||
|
|
"is_world_process_zero": true,
|
||
|
|
"log_history": [
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"completions/clipped_ratio": 0.0185546875,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 941.5,
|
||
|
|
"completions/mean_length": 228.861328125,
|
||
|
|
"completions/mean_terminated_length": 213.82315063476562,
|
||
|
|
"completions/min_length": 14.5,
|
||
|
|
"completions/min_terminated_length": 14.5,
|
||
|
|
"entropy": 0.9906870043940015,
|
||
|
|
"epoch": 0.0005334281650071124,
|
||
|
|
"frac_reward_zero_std": 0.8515625,
|
||
|
|
"grad_norm": 0.16291454434394836,
|
||
|
|
"kl": 0.0,
|
||
|
|
"learning_rate": 3.571428571428571e-08,
|
||
|
|
"loss": 0.0119,
|
||
|
|
"num_tokens": 390482.0,
|
||
|
|
"reward": 0.021484375,
|
||
|
|
"reward_std": 0.05504363030195236,
|
||
|
|
"rewards/equation_reward_func/mean": 0.001953125,
|
||
|
|
"rewards/equation_reward_func/std": 0.03121940791606903,
|
||
|
|
"rewards/format_reward_func/mean": 0.01953125,
|
||
|
|
"rewards/format_reward_func/std": 0.13361797481775284,
|
||
|
|
"sampling/importance_sampling_ratio/max": 2.0,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999333918094635,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.44458194077014923,
|
||
|
|
"sampling/sampling_logp_difference/max": 1.0505925416946411,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.02098759263753891,
|
||
|
|
"step": 2
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 5.300016507842681e-05,
|
||
|
|
"clip_ratio/high_mean": 5.300016507842681e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00010970261913219777,
|
||
|
|
"clip_ratio/low_min": 0.00010970261913219777,
|
||
|
|
"clip_ratio/region_mean": 0.00016270278421062458,
|
||
|
|
"completions/clipped_ratio": 0.02734375,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 966.0,
|
||
|
|
"completions/mean_length": 248.947265625,
|
||
|
|
"completions/mean_terminated_length": 227.15863037109375,
|
||
|
|
"completions/min_length": 20.0,
|
||
|
|
"completions/min_terminated_length": 20.0,
|
||
|
|
"entropy": 0.9930441313319736,
|
||
|
|
"epoch": 0.0010668563300142249,
|
||
|
|
"frac_reward_zero_std": 0.8125,
|
||
|
|
"grad_norm": 0.2261732518672943,
|
||
|
|
"kl": 0.0007615485147754145,
|
||
|
|
"learning_rate": 1.0714285714285713e-07,
|
||
|
|
"loss": 0.0266,
|
||
|
|
"num_tokens": 596199.0,
|
||
|
|
"reward": 0.0234375,
|
||
|
|
"reward_std": 0.06629125773906708,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0,
|
||
|
|
"rewards/equation_reward_func/std": 0.0,
|
||
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
||
|
|
"rewards/format_reward_func/std": 0.15143637359142303,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7380328178405762,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.999877393245697,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4712234437465668,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7524229288101196,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.020538652315735817,
|
||
|
|
"step": 4
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0001006675087814478,
|
||
|
|
"clip_ratio/low_min": 0.0001006675087814478,
|
||
|
|
"clip_ratio/region_mean": 0.0001006675087814478,
|
||
|
|
"completions/clipped_ratio": 0.021484375,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 1005.0,
|
||
|
|
"completions/mean_length": 243.634765625,
|
||
|
|
"completions/mean_terminated_length": 226.50099182128906,
|
||
|
|
"completions/min_length": 19.0,
|
||
|
|
"completions/min_terminated_length": 19.0,
|
||
|
|
"entropy": 0.9561915049950281,
|
||
|
|
"epoch": 0.001600284495021337,
|
||
|
|
"frac_reward_zero_std": 0.828125,
|
||
|
|
"grad_norm": 0.19601207971572876,
|
||
|
|
"kl": 0.0007531659551508104,
|
||
|
|
"learning_rate": 1.7857142857142858e-07,
|
||
|
|
"loss": 0.0277,
|
||
|
|
"num_tokens": 798924.0,
|
||
|
|
"reward": 0.03125,
|
||
|
|
"reward_std": 0.0684555172920227,
|
||
|
|
"rewards/equation_reward_func/mean": 0.001953125,
|
||
|
|
"rewards/equation_reward_func/std": 0.04419417306780815,
|
||
|
|
"rewards/format_reward_func/mean": 0.029296875,
|
||
|
|
"rewards/format_reward_func/std": 0.16880230605602264,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6818383932113647,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998596906661987,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5696869492530823,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5626683235168457,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.02069748565554619,
|
||
|
|
"step": 6
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 3.0059856928548674e-05,
|
||
|
|
"clip_ratio/low_min": 3.0059856928548674e-05,
|
||
|
|
"clip_ratio/region_mean": 3.0059856928548674e-05,
|
||
|
|
"completions/clipped_ratio": 0.01953125,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 1003.0,
|
||
|
|
"completions/mean_length": 228.125,
|
||
|
|
"completions/mean_terminated_length": 212.2709197998047,
|
||
|
|
"completions/min_length": 16.0,
|
||
|
|
"completions/min_terminated_length": 16.0,
|
||
|
|
"entropy": 0.9969107690784667,
|
||
|
|
"epoch": 0.0021337126600284497,
|
||
|
|
"frac_reward_zero_std": 0.921875,
|
||
|
|
"grad_norm": 0.17554284632205963,
|
||
|
|
"kl": 0.0007573424453666019,
|
||
|
|
"learning_rate": 2.5e-07,
|
||
|
|
"loss": 0.0153,
|
||
|
|
"num_tokens": 993724.0,
|
||
|
|
"reward": 0.009765625,
|
||
|
|
"reward_std": 0.027621358633041382,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0,
|
||
|
|
"rewards/equation_reward_func/std": 0.0,
|
||
|
|
"rewards/format_reward_func/mean": 0.009765625,
|
||
|
|
"rewards/format_reward_func/std": 0.09843364357948303,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7989540100097656,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.992868185043335,
|
||
|
|
"sampling/importance_sampling_ratio/min": 3.239048959602364e-11,
|
||
|
|
"sampling/sampling_logp_difference/max": 24.153156280517578,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.05915800854563713,
|
||
|
|
"step": 8
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 2.6382262553347067e-05,
|
||
|
|
"clip_ratio/high_mean": 2.6382262553347067e-05,
|
||
|
|
"clip_ratio/low_mean": 6.268165695170562e-05,
|
||
|
|
"clip_ratio/low_min": 6.268165695170562e-05,
|
||
|
|
"clip_ratio/region_mean": 8.906391950505268e-05,
|
||
|
|
"completions/clipped_ratio": 0.025390625,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 985.0,
|
||
|
|
"completions/mean_length": 236.822265625,
|
||
|
|
"completions/mean_terminated_length": 216.31463623046875,
|
||
|
|
"completions/min_length": 12.0,
|
||
|
|
"completions/min_terminated_length": 12.0,
|
||
|
|
"entropy": 1.0309493674172296,
|
||
|
|
"epoch": 0.0026671408250355617,
|
||
|
|
"frac_reward_zero_std": 0.828125,
|
||
|
|
"grad_norm": 0.2110099196434021,
|
||
|
|
"kl": 0.000838718113502384,
|
||
|
|
"learning_rate": 3.2142857142857145e-07,
|
||
|
|
"loss": 0.0136,
|
||
|
|
"num_tokens": 1193225.0,
|
||
|
|
"reward": 0.0234375,
|
||
|
|
"reward_std": 0.06247568875551224,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0,
|
||
|
|
"rewards/equation_reward_func/std": 0.0,
|
||
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
||
|
|
"rewards/format_reward_func/std": 0.15143637359142303,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8266597986221313,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000933408737183,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4933336079120636,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7065696716308594,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.020812492817640305,
|
||
|
|
"step": 10
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0001034049900125764,
|
||
|
|
"clip_ratio/low_min": 0.0001034049900125764,
|
||
|
|
"clip_ratio/region_mean": 0.0001034049900125764,
|
||
|
|
"completions/clipped_ratio": 0.03125,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 987.0,
|
||
|
|
"completions/mean_length": 230.103515625,
|
||
|
|
"completions/mean_terminated_length": 204.4939422607422,
|
||
|
|
"completions/min_length": 13.0,
|
||
|
|
"completions/min_terminated_length": 13.0,
|
||
|
|
"entropy": 1.0229314300749037,
|
||
|
|
"epoch": 0.003200568990042674,
|
||
|
|
"frac_reward_zero_std": 0.890625,
|
||
|
|
"grad_norm": 0.1713971197605133,
|
||
|
|
"kl": 0.0007667660134352951,
|
||
|
|
"learning_rate": 3.928571428571428e-07,
|
||
|
|
"loss": 0.0199,
|
||
|
|
"num_tokens": 1389166.0,
|
||
|
|
"reward": 0.015625,
|
||
|
|
"reward_std": 0.04037860035896301,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0,
|
||
|
|
"rewards/equation_reward_func/std": 0.0,
|
||
|
|
"rewards/format_reward_func/mean": 0.015625,
|
||
|
|
"rewards/format_reward_func/std": 0.12414088100194931,
|
||
|
|
"sampling/importance_sampling_ratio/max": 2.0,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999401569366455,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4219718873500824,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.9450585842132568,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.02105509303510189,
|
||
|
|
"step": 12
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 8.771006832830608e-06,
|
||
|
|
"clip_ratio/high_mean": 8.771006832830608e-06,
|
||
|
|
"clip_ratio/low_mean": 9.435887235061576e-05,
|
||
|
|
"clip_ratio/low_min": 9.435887235061576e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00010312987918344636,
|
||
|
|
"completions/clipped_ratio": 0.033203125,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 998.0,
|
||
|
|
"completions/mean_length": 237.861328125,
|
||
|
|
"completions/mean_terminated_length": 210.86264038085938,
|
||
|
|
"completions/min_length": 15.0,
|
||
|
|
"completions/min_terminated_length": 15.0,
|
||
|
|
"entropy": 0.9931543800565932,
|
||
|
|
"epoch": 0.0037339971550497866,
|
||
|
|
"frac_reward_zero_std": 0.84375,
|
||
|
|
"grad_norm": 0.18590842187404633,
|
||
|
|
"kl": 0.0008038054689273445,
|
||
|
|
"learning_rate": 4.6428571428571427e-07,
|
||
|
|
"loss": 0.0143,
|
||
|
|
"num_tokens": 1589071.0,
|
||
|
|
"reward": 0.01953125,
|
||
|
|
"reward_std": 0.055242717266082764,
|
||
|
|
"rewards/equation_reward_func/mean": 0.001953125,
|
||
|
|
"rewards/equation_reward_func/std": 0.04419417306780815,
|
||
|
|
"rewards/format_reward_func/mean": 0.017578125,
|
||
|
|
"rewards/format_reward_func/std": 0.13154059648513794,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8855645656585693,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999428391456604,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.2005501091480255,
|
||
|
|
"sampling/sampling_logp_difference/max": 1.6066911220550537,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.020327817648649216,
|
||
|
|
"step": 14
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 9.463882532953803e-05,
|
||
|
|
"clip_ratio/low_min": 9.463882532953803e-05,
|
||
|
|
"clip_ratio/region_mean": 9.463882532953803e-05,
|
||
|
|
"completions/clipped_ratio": 0.029296875,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 984.0,
|
||
|
|
"completions/mean_length": 250.32421875,
|
||
|
|
"completions/mean_terminated_length": 226.9738311767578,
|
||
|
|
"completions/min_length": 14.0,
|
||
|
|
"completions/min_terminated_length": 14.0,
|
||
|
|
"entropy": 0.9750576963027319,
|
||
|
|
"epoch": 0.004267425320056899,
|
||
|
|
"frac_reward_zero_std": 0.828125,
|
||
|
|
"grad_norm": 0.19739331305027008,
|
||
|
|
"kl": 0.0010235626525374958,
|
||
|
|
"learning_rate": 4.999935101463869e-07,
|
||
|
|
"loss": 0.0143,
|
||
|
|
"num_tokens": 1795301.0,
|
||
|
|
"reward": 0.0234375,
|
||
|
|
"reward_std": 0.06247568875551224,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0,
|
||
|
|
"rewards/equation_reward_func/std": 0.0,
|
||
|
|
"rewards/format_reward_func/mean": 0.0234375,
|
||
|
|
"rewards/format_reward_func/std": 0.15143637359142303,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8375256061553955,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9952285885810852,
|
||
|
|
"sampling/importance_sampling_ratio/min": 7.352532471749385e-11,
|
||
|
|
"sampling/sampling_logp_difference/max": 23.333391189575195,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.07141915708780289,
|
||
|
|
"step": 16
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 7.348618560677602e-06,
|
||
|
|
"clip_ratio/high_mean": 7.348618560677602e-06,
|
||
|
|
"clip_ratio/low_mean": 6.096324634401956e-05,
|
||
|
|
"clip_ratio/low_min": 6.096324634401956e-05,
|
||
|
|
"clip_ratio/region_mean": 6.831186490469716e-05,
|
||
|
|
"completions/clipped_ratio": 0.0224609375,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 1007.0,
|
||
|
|
"completions/mean_length": 231.904296875,
|
||
|
|
"completions/mean_terminated_length": 213.72539520263672,
|
||
|
|
"completions/min_length": 15.0,
|
||
|
|
"completions/min_terminated_length": 15.0,
|
||
|
|
"entropy": 1.015358070532481,
|
||
|
|
"epoch": 0.004800853485064011,
|
||
|
|
"frac_reward_zero_std": 0.6953125,
|
||
|
|
"grad_norm": 0.31367021799087524,
|
||
|
|
"kl": 0.0011365925408123683,
|
||
|
|
"learning_rate": 4.999415933391384e-07,
|
||
|
|
"loss": 0.0309,
|
||
|
|
"num_tokens": 2188979.0,
|
||
|
|
"reward": 0.0458984375,
|
||
|
|
"reward_std": 0.11455809697508812,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0029296875,
|
||
|
|
"rewards/equation_reward_func/std": 0.053316494449973106,
|
||
|
|
"rewards/format_reward_func/mean": 0.04296875,
|
||
|
|
"rewards/format_reward_func/std": 0.20246700942516327,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.9270648956298828,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000028133392334,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.23970969766378403,
|
||
|
|
"sampling/sampling_logp_difference/max": 1.4344506859779358,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.02102941181510687,
|
||
|
|
"step": 18
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 8.455857298233443e-05,
|
||
|
|
"clip_ratio/high_mean": 8.455857298233443e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00023414362092605897,
|
||
|
|
"clip_ratio/low_min": 0.00023414362092605897,
|
||
|
|
"clip_ratio/region_mean": 0.0003187021922915139,
|
||
|
|
"completions/clipped_ratio": 0.009765625,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 928.0,
|
||
|
|
"completions/mean_length": 208.12109375,
|
||
|
|
"completions/mean_terminated_length": 200.074951171875,
|
||
|
|
"completions/min_length": 14.0,
|
||
|
|
"completions/min_terminated_length": 14.0,
|
||
|
|
"entropy": 0.9739413443538878,
|
||
|
|
"epoch": 0.005334281650071123,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.29864609241485596,
|
||
|
|
"kl": 0.0018909820646513253,
|
||
|
|
"learning_rate": 4.998377705063407e-07,
|
||
|
|
"loss": 0.043,
|
||
|
|
"num_tokens": 2373873.0,
|
||
|
|
"reward": 0.05859375,
|
||
|
|
"reward_std": 0.1165214255452156,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0,
|
||
|
|
"rewards/equation_reward_func/std": 0.0,
|
||
|
|
"rewards/format_reward_func/mean": 0.05859375,
|
||
|
|
"rewards/format_reward_func/std": 0.23509246110916138,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7688930034637451,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998641610145569,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5252137184143066,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6439499855041504,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.020961318165063858,
|
||
|
|
"step": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 8.217327882044224e-05,
|
||
|
|
"clip_ratio/high_mean": 8.217327882044224e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00020888712970190682,
|
||
|
|
"clip_ratio/low_min": 0.00020888712970190682,
|
||
|
|
"clip_ratio/region_mean": 0.00029106041256454773,
|
||
|
|
"completions/clipped_ratio": 0.017578125,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 1006.0,
|
||
|
|
"completions/mean_length": 217.525390625,
|
||
|
|
"completions/mean_terminated_length": 203.0954132080078,
|
||
|
|
"completions/min_length": 13.0,
|
||
|
|
"completions/min_terminated_length": 13.0,
|
||
|
|
"entropy": 0.9706550488869349,
|
||
|
|
"epoch": 0.005867709815078236,
|
||
|
|
"frac_reward_zero_std": 0.515625,
|
||
|
|
"grad_norm": 0.32972589135169983,
|
||
|
|
"kl": 0.0031107893123084474,
|
||
|
|
"learning_rate": 4.996820632091536e-07,
|
||
|
|
"loss": 0.0343,
|
||
|
|
"num_tokens": 2563390.0,
|
||
|
|
"reward": 0.078125,
|
||
|
|
"reward_std": 0.1843322515487671,
|
||
|
|
"rewards/equation_reward_func/mean": 0.00390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.06243881583213806,
|
||
|
|
"rewards/format_reward_func/mean": 0.07421875,
|
||
|
|
"rewards/format_reward_func/std": 0.2623828947544098,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.894946575164795,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999268054962158,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.11174897849559784,
|
||
|
|
"sampling/sampling_logp_difference/max": 2.191500186920166,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.021253950893878937,
|
||
|
|
"step": 22
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.2112408209488625e-05,
|
||
|
|
"clip_ratio/high_mean": 3.2112408209488625e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00027551787813232903,
|
||
|
|
"clip_ratio/low_min": 0.00027551787813232903,
|
||
|
|
"clip_ratio/region_mean": 0.00030763028634181764,
|
||
|
|
"completions/clipped_ratio": 0.0234375,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 1014.0,
|
||
|
|
"completions/mean_length": 194.970703125,
|
||
|
|
"completions/mean_terminated_length": 175.07400512695312,
|
||
|
|
"completions/min_length": 14.0,
|
||
|
|
"completions/min_terminated_length": 14.0,
|
||
|
|
"entropy": 0.9849063654740652,
|
||
|
|
"epoch": 0.006401137980085348,
|
||
|
|
"frac_reward_zero_std": 0.328125,
|
||
|
|
"grad_norm": 0.6001269817352295,
|
||
|
|
"kl": 0.003953174649116893,
|
||
|
|
"learning_rate": 4.994745037837194e-07,
|
||
|
|
"loss": 0.0869,
|
||
|
|
"num_tokens": 2741335.0,
|
||
|
|
"reward": 0.126953125,
|
||
|
|
"reward_std": 0.2717152237892151,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0,
|
||
|
|
"rewards/equation_reward_func/std": 0.0,
|
||
|
|
"rewards/format_reward_func/mean": 0.126953125,
|
||
|
|
"rewards/format_reward_func/std": 0.33324605226516724,
|
||
|
|
"sampling/importance_sampling_ratio/max": 2.0,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998724460601807,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4356817901134491,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8308432102203369,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.02183392085134983,
|
||
|
|
"step": 24
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0003703655949923106,
|
||
|
|
"clip_ratio/high_mean": 0.0003703655949923106,
|
||
|
|
"clip_ratio/low_mean": 0.0029490124684849028,
|
||
|
|
"clip_ratio/low_min": 0.0029490124684849028,
|
||
|
|
"clip_ratio/region_mean": 0.0033193781152173565,
|
||
|
|
"completions/clipped_ratio": 0.013671875,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 983.0,
|
||
|
|
"completions/mean_length": 180.943359375,
|
||
|
|
"completions/mean_terminated_length": 169.25743103027344,
|
||
|
|
"completions/min_length": 11.0,
|
||
|
|
"completions/min_terminated_length": 11.0,
|
||
|
|
"entropy": 1.019554615020752,
|
||
|
|
"epoch": 0.006934566145092461,
|
||
|
|
"frac_reward_zero_std": 0.21875,
|
||
|
|
"grad_norm": 0.8571369051933289,
|
||
|
|
"kl": 0.010997309517632756,
|
||
|
|
"learning_rate": 4.992151353344481e-07,
|
||
|
|
"loss": 0.0869,
|
||
|
|
"num_tokens": 2912138.0,
|
||
|
|
"reward": 0.171875,
|
||
|
|
"reward_std": 0.3274696171283722,
|
||
|
|
"rewards/equation_reward_func/mean": 0.001953125,
|
||
|
|
"rewards/equation_reward_func/std": 0.04419417306780815,
|
||
|
|
"rewards/format_reward_func/mean": 0.169921875,
|
||
|
|
"rewards/format_reward_func/std": 0.3759314715862274,
|
||
|
|
"sampling/importance_sampling_ratio/max": 2.0,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9895955324172974,
|
||
|
|
"sampling/importance_sampling_ratio/min": 5.196759342296353e-11,
|
||
|
|
"sampling/sampling_logp_difference/max": 23.680400848388672,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.11238687485456467,
|
||
|
|
"step": 26
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0007882783518450904,
|
||
|
|
"clip_ratio/high_mean": 0.0007882783518450904,
|
||
|
|
"clip_ratio/low_mean": 0.00845093373532614,
|
||
|
|
"clip_ratio/low_min": 0.00845093373532614,
|
||
|
|
"clip_ratio/region_mean": 0.009239212098893605,
|
||
|
|
"completions/clipped_ratio": 0.0078125,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 1001.0,
|
||
|
|
"completions/mean_length": 156.486328125,
|
||
|
|
"completions/mean_terminated_length": 149.655517578125,
|
||
|
|
"completions/min_length": 14.0,
|
||
|
|
"completions/min_terminated_length": 14.0,
|
||
|
|
"entropy": 0.9848703841368357,
|
||
|
|
"epoch": 0.007467994310099573,
|
||
|
|
"frac_reward_zero_std": 0.078125,
|
||
|
|
"grad_norm": 0.9129703640937805,
|
||
|
|
"kl": 0.0329602628480643,
|
||
|
|
"learning_rate": 4.989040117250646e-07,
|
||
|
|
"loss": 0.1433,
|
||
|
|
"num_tokens": 3070395.0,
|
||
|
|
"reward": 0.279296875,
|
||
|
|
"reward_std": 0.42425647377967834,
|
||
|
|
"rewards/equation_reward_func/mean": 0.00390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.06243881583213806,
|
||
|
|
"rewards/format_reward_func/mean": 0.275390625,
|
||
|
|
"rewards/format_reward_func/std": 0.44714778661727905,
|
||
|
|
"sampling/importance_sampling_ratio/max": 2.0,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9907291531562805,
|
||
|
|
"sampling/importance_sampling_ratio/min": 3.045626945086255e-11,
|
||
|
|
"sampling/sampling_logp_difference/max": 24.21472930908203,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.1076340302824974,
|
||
|
|
"step": 28
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0006181110511533916,
|
||
|
|
"clip_ratio/high_mean": 0.0006181110511533916,
|
||
|
|
"clip_ratio/low_mean": 0.002150450627798111,
|
||
|
|
"clip_ratio/low_min": 0.002150450627798111,
|
||
|
|
"clip_ratio/region_mean": 0.0027685616951202974,
|
||
|
|
"completions/clipped_ratio": 0.0078125,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 888.0,
|
||
|
|
"completions/mean_length": 127.173828125,
|
||
|
|
"completions/mean_terminated_length": 120.1122055053711,
|
||
|
|
"completions/min_length": 18.0,
|
||
|
|
"completions/min_terminated_length": 18.0,
|
||
|
|
"entropy": 0.902042117383745,
|
||
|
|
"epoch": 0.008001422475106686,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 0.9917025566101074,
|
||
|
|
"kl": 0.026186900937722787,
|
||
|
|
"learning_rate": 4.985411975674243e-07,
|
||
|
|
"loss": 0.1706,
|
||
|
|
"num_tokens": 3213676.0,
|
||
|
|
"reward": 0.458984375,
|
||
|
|
"reward_std": 0.4996362030506134,
|
||
|
|
"rewards/equation_reward_func/mean": 0.00390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.06243881583213806,
|
||
|
|
"rewards/format_reward_func/mean": 0.455078125,
|
||
|
|
"rewards/format_reward_func/std": 0.4984649419784546,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.58962881565094,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000369548797607,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5563350915908813,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5863845348358154,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.020490314811468124,
|
||
|
|
"step": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0004448202024731371,
|
||
|
|
"clip_ratio/high_mean": 0.0004448202024731371,
|
||
|
|
"clip_ratio/low_mean": 0.0014546795913095896,
|
||
|
|
"clip_ratio/low_min": 0.0014546795913095896,
|
||
|
|
"clip_ratio/region_mean": 0.0018994997986333652,
|
||
|
|
"completions/clipped_ratio": 0.005859375,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 801.0,
|
||
|
|
"completions/mean_length": 117.958984375,
|
||
|
|
"completions/mean_terminated_length": 112.61886596679688,
|
||
|
|
"completions/min_length": 18.0,
|
||
|
|
"completions/min_terminated_length": 18.0,
|
||
|
|
"entropy": 0.8814441644483142,
|
||
|
|
"epoch": 0.008534850640113799,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"grad_norm": 1.08668053150177,
|
||
|
|
"kl": 0.033620989260574184,
|
||
|
|
"learning_rate": 4.981267682080939e-07,
|
||
|
|
"loss": 0.2037,
|
||
|
|
"num_tokens": 3352399.0,
|
||
|
|
"reward": 0.544921875,
|
||
|
|
"reward_std": 0.5004862546920776,
|
||
|
|
"rewards/equation_reward_func/mean": 0.00390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.06243881583213806,
|
||
|
|
"rewards/format_reward_func/mean": 0.541015625,
|
||
|
|
"rewards/format_reward_func/std": 0.49880221486091614,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.592621088027954,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0002690553665161,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.627295732498169,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4663372039794922,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.020882319658994675,
|
||
|
|
"step": 32
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00021157415337964066,
|
||
|
|
"clip_ratio/high_mean": 0.00021157415337964066,
|
||
|
|
"clip_ratio/low_mean": 0.0010422569240391669,
|
||
|
|
"clip_ratio/low_min": 0.0010422569240391669,
|
||
|
|
"clip_ratio/region_mean": 0.0012538310580162539,
|
||
|
|
"completions/clipped_ratio": 0.0029296875,
|
||
|
|
"completions/max_length": 932.5,
|
||
|
|
"completions/max_terminated_length": 879.0,
|
||
|
|
"completions/mean_length": 97.865234375,
|
||
|
|
"completions/mean_terminated_length": 95.14189910888672,
|
||
|
|
"completions/min_length": 17.5,
|
||
|
|
"completions/min_terminated_length": 17.5,
|
||
|
|
"entropy": 0.8883479121658537,
|
||
|
|
"epoch": 0.00906827880512091,
|
||
|
|
"frac_reward_zero_std": 0.046875,
|
||
|
|
"grad_norm": 1.2837241888046265,
|
||
|
|
"kl": 0.049437474925071,
|
||
|
|
"learning_rate": 4.976608097127043e-07,
|
||
|
|
"loss": 0.1989,
|
||
|
|
"num_tokens": 3608821.0,
|
||
|
|
"reward": 0.666015625,
|
||
|
|
"reward_std": 0.4548095315694809,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0029296875,
|
||
|
|
"rewards/equation_reward_func/std": 0.053316494449973106,
|
||
|
|
"rewards/format_reward_func/mean": 0.6630859375,
|
||
|
|
"rewards/format_reward_func/std": 0.4725191593170166,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6358660459518433,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9954747259616852,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.2847309411318844,
|
||
|
|
"sampling/sampling_logp_difference/max": 11.593867659568787,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.07399317715317011,
|
||
|
|
"step": 34
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0006300570425486916,
|
||
|
|
"clip_ratio/high_mean": 0.0006300570425486916,
|
||
|
|
"clip_ratio/low_mean": 0.004105484530252094,
|
||
|
|
"clip_ratio/low_min": 0.004105484530252094,
|
||
|
|
"clip_ratio/region_mean": 0.004735541625349369,
|
||
|
|
"completions/clipped_ratio": 0.005859375,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 845.0,
|
||
|
|
"completions/mean_length": 95.708984375,
|
||
|
|
"completions/mean_terminated_length": 90.23772430419922,
|
||
|
|
"completions/min_length": 25.0,
|
||
|
|
"completions/min_terminated_length": 25.0,
|
||
|
|
"entropy": 0.8173519985543357,
|
||
|
|
"epoch": 0.009601706970128023,
|
||
|
|
"frac_reward_zero_std": 0.046875,
|
||
|
|
"grad_norm": 1.169929027557373,
|
||
|
|
"kl": 0.07463782078896959,
|
||
|
|
"learning_rate": 4.97143418848077e-07,
|
||
|
|
"loss": 0.1797,
|
||
|
|
"num_tokens": 3736000.0,
|
||
|
|
"reward": 0.75,
|
||
|
|
"reward_std": 0.4411673843860626,
|
||
|
|
"rewards/equation_reward_func/mean": 0.005859375,
|
||
|
|
"rewards/equation_reward_func/std": 0.07639661431312561,
|
||
|
|
"rewards/format_reward_func/mean": 0.744140625,
|
||
|
|
"rewards/format_reward_func/std": 0.43676990270614624,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7114160060882568,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9796027541160583,
|
||
|
|
"sampling/importance_sampling_ratio/min": 5.638221139969701e-11,
|
||
|
|
"sampling/sampling_logp_difference/max": 23.598867416381836,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.26203441619873047,
|
||
|
|
"step": 36
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0004559664189906067,
|
||
|
|
"clip_ratio/high_mean": 0.0004559664189906067,
|
||
|
|
"clip_ratio/low_mean": 0.00839672572813773,
|
||
|
|
"clip_ratio/low_min": 0.00839672572813773,
|
||
|
|
"clip_ratio/region_mean": 0.008852692146319896,
|
||
|
|
"completions/clipped_ratio": 0.001953125,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 706.0,
|
||
|
|
"completions/mean_length": 75.271484375,
|
||
|
|
"completions/mean_terminated_length": 73.41487121582031,
|
||
|
|
"completions/min_length": 25.0,
|
||
|
|
"completions/min_terminated_length": 25.0,
|
||
|
|
"entropy": 0.8584042108721204,
|
||
|
|
"epoch": 0.010135135135135136,
|
||
|
|
"frac_reward_zero_std": 0.28125,
|
||
|
|
"grad_norm": 1.4801753759384155,
|
||
|
|
"kl": 28.380483800234895,
|
||
|
|
"learning_rate": 4.965747030621286e-07,
|
||
|
|
"loss": 0.2329,
|
||
|
|
"num_tokens": 3852635.0,
|
||
|
|
"reward": 0.865234375,
|
||
|
|
"reward_std": 0.2919681668281555,
|
||
|
|
"rewards/equation_reward_func/mean": 0.00390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.06243881583213806,
|
||
|
|
"rewards/format_reward_func/mean": 0.861328125,
|
||
|
|
"rewards/format_reward_func/std": 0.34594178199768066,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.733668327331543,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000662803649902,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4352138638496399,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8319177627563477,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.01950383372604847,
|
||
|
|
"step": 38
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0004300644068280235,
|
||
|
|
"clip_ratio/high_mean": 0.0004300644068280235,
|
||
|
|
"clip_ratio/low_mean": 0.0008942384576787137,
|
||
|
|
"clip_ratio/low_min": 0.0008942384576787137,
|
||
|
|
"clip_ratio/region_mean": 0.0013243028612729784,
|
||
|
|
"completions/clipped_ratio": 0.00390625,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 604.0,
|
||
|
|
"completions/mean_length": 74.28515625,
|
||
|
|
"completions/mean_terminated_length": 70.560791015625,
|
||
|
|
"completions/min_length": 21.0,
|
||
|
|
"completions/min_terminated_length": 21.0,
|
||
|
|
"entropy": 0.828421617547671,
|
||
|
|
"epoch": 0.010668563300142247,
|
||
|
|
"frac_reward_zero_std": 0.5,
|
||
|
|
"grad_norm": 266.15692138671875,
|
||
|
|
"kl": 10.770809243329698,
|
||
|
|
"learning_rate": 4.959547804615562e-07,
|
||
|
|
"loss": 0.1596,
|
||
|
|
"num_tokens": 3968701.0,
|
||
|
|
"reward": 0.91015625,
|
||
|
|
"reward_std": 0.2011794149875641,
|
||
|
|
"rewards/equation_reward_func/mean": 0.001953125,
|
||
|
|
"rewards/equation_reward_func/std": 0.04419417306780815,
|
||
|
|
"rewards/format_reward_func/mean": 0.908203125,
|
||
|
|
"rewards/format_reward_func/std": 0.289021372795105,
|
||
|
|
"sampling/importance_sampling_ratio/max": 2.0,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9749342203140259,
|
||
|
|
"sampling/importance_sampling_ratio/min": 7.815229713381555e-11,
|
||
|
|
"sampling/sampling_logp_difference/max": 23.272361755371094,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.30028611421585083,
|
||
|
|
"step": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0007337608694797382,
|
||
|
|
"clip_ratio/high_mean": 0.0007337608694797382,
|
||
|
|
"clip_ratio/low_mean": 0.000700917316458395,
|
||
|
|
"clip_ratio/low_min": 0.000700917316458395,
|
||
|
|
"clip_ratio/region_mean": 0.0014346781843212536,
|
||
|
|
"completions/clipped_ratio": 0.00390625,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 995.0,
|
||
|
|
"completions/mean_length": 73.091796875,
|
||
|
|
"completions/mean_terminated_length": 69.36274719238281,
|
||
|
|
"completions/min_length": 26.0,
|
||
|
|
"completions/min_terminated_length": 26.0,
|
||
|
|
"entropy": 0.7282936986949708,
|
||
|
|
"epoch": 0.01120199146514936,
|
||
|
|
"frac_reward_zero_std": 0.515625,
|
||
|
|
"grad_norm": 1.3456498384475708,
|
||
|
|
"kl": 0.0651899280750917,
|
||
|
|
"learning_rate": 4.952837797873106e-07,
|
||
|
|
"loss": 0.1532,
|
||
|
|
"num_tokens": 4084332.0,
|
||
|
|
"reward": 0.93359375,
|
||
|
|
"reward_std": 0.1842852383852005,
|
||
|
|
"rewards/equation_reward_func/mean": 0.00390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.06243881583213806,
|
||
|
|
"rewards/format_reward_func/mean": 0.9296875,
|
||
|
|
"rewards/format_reward_func/std": 0.25592297315597534,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4386967420578003,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9996576309204102,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.24208980798721313,
|
||
|
|
"sampling/sampling_logp_difference/max": 1.4184465408325195,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.018967146053910255,
|
||
|
|
"step": 42
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0005429589736094284,
|
||
|
|
"clip_ratio/high_mean": 0.0005429589736094284,
|
||
|
|
"clip_ratio/low_mean": 0.0007054127959741487,
|
||
|
|
"clip_ratio/low_min": 0.0007054127959741487,
|
||
|
|
"clip_ratio/region_mean": 0.0012483717776679744,
|
||
|
|
"completions/clipped_ratio": 0.00390625,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 765.0,
|
||
|
|
"completions/mean_length": 65.826171875,
|
||
|
|
"completions/mean_terminated_length": 62.06863021850586,
|
||
|
|
"completions/min_length": 17.0,
|
||
|
|
"completions/min_terminated_length": 17.0,
|
||
|
|
"entropy": 0.6828638480769263,
|
||
|
|
"epoch": 0.011735419630156473,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 1.0256437063217163,
|
||
|
|
"kl": 0.09110636295129855,
|
||
|
|
"learning_rate": 4.9456184038786e-07,
|
||
|
|
"loss": 0.141,
|
||
|
|
"num_tokens": 4196131.0,
|
||
|
|
"reward": 0.970703125,
|
||
|
|
"reward_std": 0.10905230790376663,
|
||
|
|
"rewards/equation_reward_func/mean": 0.009765625,
|
||
|
|
"rewards/equation_reward_func/std": 0.09843364357948303,
|
||
|
|
"rewards/format_reward_func/mean": 0.9609375,
|
||
|
|
"rewards/format_reward_func/std": 0.1939331740140915,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4373838901519775,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9722147583961487,
|
||
|
|
"sampling/importance_sampling_ratio/min": 5.824247334418331e-11,
|
||
|
|
"sampling/sampling_logp_difference/max": 23.56640625,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.2027144581079483,
|
||
|
|
"step": 44
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.012247134071205638,
|
||
|
|
"clip_ratio/high_mean": 0.012247134071205638,
|
||
|
|
"clip_ratio/low_mean": 0.000293311591020837,
|
||
|
|
"clip_ratio/low_min": 0.000293311591020837,
|
||
|
|
"clip_ratio/region_mean": 0.012540445503772289,
|
||
|
|
"completions/clipped_ratio": 0.00390625,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 561.0,
|
||
|
|
"completions/mean_length": 64.818359375,
|
||
|
|
"completions/mean_terminated_length": 61.05686569213867,
|
||
|
|
"completions/min_length": 22.0,
|
||
|
|
"completions/min_terminated_length": 22.0,
|
||
|
|
"entropy": 0.7212139268716177,
|
||
|
|
"epoch": 0.012268847795163585,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 1.3760632276535034,
|
||
|
|
"kl": 1.7077488615694973,
|
||
|
|
"learning_rate": 4.937891121902508e-07,
|
||
|
|
"loss": 0.1133,
|
||
|
|
"num_tokens": 4307502.0,
|
||
|
|
"reward": 0.966796875,
|
||
|
|
"reward_std": 0.13146039843559265,
|
||
|
|
"rewards/equation_reward_func/mean": 0.009765625,
|
||
|
|
"rewards/equation_reward_func/std": 0.09843364357948303,
|
||
|
|
"rewards/format_reward_func/mean": 0.95703125,
|
||
|
|
"rewards/format_reward_func/std": 0.2029850035905838,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4471436738967896,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9727644920349121,
|
||
|
|
"sampling/importance_sampling_ratio/min": 6.65283175771414e-11,
|
||
|
|
"sampling/sampling_logp_difference/max": 23.433393478393555,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.1866401731967926,
|
||
|
|
"step": 46
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0002938441725240813,
|
||
|
|
"clip_ratio/high_mean": 0.0002938441725240813,
|
||
|
|
"clip_ratio/low_mean": 0.0002586909201151381,
|
||
|
|
"clip_ratio/low_min": 0.0002586909201151381,
|
||
|
|
"clip_ratio/region_mean": 0.0005525350894054605,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 785.0,
|
||
|
|
"completions/max_terminated_length": 785.0,
|
||
|
|
"completions/mean_length": 55.564453125,
|
||
|
|
"completions/mean_terminated_length": 55.564453125,
|
||
|
|
"completions/min_length": 20.0,
|
||
|
|
"completions/min_terminated_length": 20.0,
|
||
|
|
"entropy": 0.6383216695653068,
|
||
|
|
"epoch": 0.012802275960170697,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.8559442162513733,
|
||
|
|
"kl": 0.1444133176571793,
|
||
|
|
"learning_rate": 4.929657556689726e-07,
|
||
|
|
"loss": 0.0403,
|
||
|
|
"num_tokens": 4413927.0,
|
||
|
|
"reward": 0.970703125,
|
||
|
|
"reward_std": 0.11219413578510284,
|
||
|
|
"rewards/equation_reward_func/mean": 0.005859375,
|
||
|
|
"rewards/equation_reward_func/std": 0.07639661431312561,
|
||
|
|
"rewards/format_reward_func/mean": 0.96484375,
|
||
|
|
"rewards/format_reward_func/std": 0.1843547374010086,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4029735326766968,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.00029456615448,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4570175111293793,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7830336093902588,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.018120447173714638,
|
||
|
|
"step": 48
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.298361449812849e-05,
|
||
|
|
"clip_ratio/high_mean": 9.298361449812849e-05,
|
||
|
|
"clip_ratio/low_mean": 0.003963445601079406,
|
||
|
|
"clip_ratio/low_min": 0.003963445601079406,
|
||
|
|
"clip_ratio/region_mean": 0.004056429215577534,
|
||
|
|
"completions/clipped_ratio": 0.001953125,
|
||
|
|
"completions/max_length": 455.5,
|
||
|
|
"completions/max_terminated_length": 455.5,
|
||
|
|
"completions/mean_length": 56.0908203125,
|
||
|
|
"completions/mean_terminated_length": 55.42214393615723,
|
||
|
|
"completions/min_length": 22.0,
|
||
|
|
"completions/min_terminated_length": 22.0,
|
||
|
|
"entropy": 0.703913731707467,
|
||
|
|
"epoch": 0.01333570412517781,
|
||
|
|
"frac_reward_zero_std": 0.8203125,
|
||
|
|
"grad_norm": 1.4378124475479126,
|
||
|
|
"kl": 0.23787370945016542,
|
||
|
|
"learning_rate": 4.920919418126312e-07,
|
||
|
|
"loss": 0.0137,
|
||
|
|
"num_tokens": 4627532.0,
|
||
|
|
"reward": 0.9814453125,
|
||
|
|
"reward_std": 0.06523782201111317,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0029296875,
|
||
|
|
"rewards/equation_reward_func/std": 0.038198307156562805,
|
||
|
|
"rewards/format_reward_func/mean": 0.978515625,
|
||
|
|
"rewards/format_reward_func/std": 0.14252030476927757,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7019571661949158,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9878095090389252,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.2683930397276962,
|
||
|
|
"sampling/sampling_logp_difference/max": 12.1842200756073,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.1233413452282548,
|
||
|
|
"step": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.745953138917685e-05,
|
||
|
|
"clip_ratio/high_mean": 9.745953138917685e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00014718516225305697,
|
||
|
|
"clip_ratio/low_min": 0.00014718516225305697,
|
||
|
|
"clip_ratio/region_mean": 0.0002446446936422338,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 441.0,
|
||
|
|
"completions/max_terminated_length": 441.0,
|
||
|
|
"completions/mean_length": 56.39453125,
|
||
|
|
"completions/mean_terminated_length": 56.39453125,
|
||
|
|
"completions/min_length": 25.0,
|
||
|
|
"completions/min_terminated_length": 25.0,
|
||
|
|
"entropy": 0.5998011049297121,
|
||
|
|
"epoch": 0.013869132290184922,
|
||
|
|
"frac_reward_zero_std": 0.859375,
|
||
|
|
"grad_norm": 0.650745153427124,
|
||
|
|
"kl": 0.11571263087292512,
|
||
|
|
"learning_rate": 4.911678520884398e-07,
|
||
|
|
"loss": 0.0339,
|
||
|
|
"num_tokens": 4734494.0,
|
||
|
|
"reward": 0.99609375,
|
||
|
|
"reward_std": 0.051427144557237625,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0078125,
|
||
|
|
"rewards/equation_reward_func/std": 0.08812850713729858,
|
||
|
|
"rewards/format_reward_func/mean": 0.98828125,
|
||
|
|
"rewards/format_reward_func/std": 0.10772226005792618,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6142765283584595,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999446868896484,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6179465651512146,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4813532829284668,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.01780065894126892,
|
||
|
|
"step": 52
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00015658765591474043,
|
||
|
|
"clip_ratio/high_mean": 0.00015658765591474043,
|
||
|
|
"clip_ratio/low_mean": 0.0001687218997782717,
|
||
|
|
"clip_ratio/low_min": 0.0001687218997782717,
|
||
|
|
"clip_ratio/region_mean": 0.00032530955569301214,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 255.0,
|
||
|
|
"completions/max_terminated_length": 255.0,
|
||
|
|
"completions/mean_length": 55.21484375,
|
||
|
|
"completions/mean_terminated_length": 55.21484375,
|
||
|
|
"completions/min_length": 22.0,
|
||
|
|
"completions/min_terminated_length": 22.0,
|
||
|
|
"entropy": 0.5713819041848183,
|
||
|
|
"epoch": 0.014402560455192033,
|
||
|
|
"frac_reward_zero_std": 0.84375,
|
||
|
|
"grad_norm": 0.9380442500114441,
|
||
|
|
"kl": 0.09363365049163501,
|
||
|
|
"learning_rate": 4.901936784045324e-07,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 4841012.0,
|
||
|
|
"reward": 1.00390625,
|
||
|
|
"reward_std": 0.055242717266082764,
|
||
|
|
"rewards/equation_reward_func/mean": 0.01171875,
|
||
|
|
"rewards/equation_reward_func/std": 0.10772226005792618,
|
||
|
|
"rewards/format_reward_func/mean": 0.9921875,
|
||
|
|
"rewards/format_reward_func/std": 0.08812850713729858,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5352394580841064,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997539520263672,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5285897254943848,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.637542724609375,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.016956791281700134,
|
||
|
|
"step": 54
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 8.734321777915789e-05,
|
||
|
|
"clip_ratio/high_mean": 8.734321777915789e-05,
|
||
|
|
"clip_ratio/low_mean": 9.36030507242928e-05,
|
||
|
|
"clip_ratio/low_min": 9.36030507242928e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00018094626850345067,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 116.0,
|
||
|
|
"completions/max_terminated_length": 116.0,
|
||
|
|
"completions/mean_length": 54.68359375,
|
||
|
|
"completions/mean_terminated_length": 54.68359375,
|
||
|
|
"completions/min_length": 26.0,
|
||
|
|
"completions/min_terminated_length": 26.0,
|
||
|
|
"entropy": 0.5553412776854303,
|
||
|
|
"epoch": 0.014935988620199146,
|
||
|
|
"frac_reward_zero_std": 0.90625,
|
||
|
|
"grad_norm": 0.4358917474746704,
|
||
|
|
"kl": 0.10994591563940048,
|
||
|
|
"learning_rate": 4.891696230701103e-07,
|
||
|
|
"loss": 0.0028,
|
||
|
|
"num_tokens": 4947034.0,
|
||
|
|
"reward": 0.998046875,
|
||
|
|
"reward_std": 0.03597327321767807,
|
||
|
|
"rewards/equation_reward_func/mean": 0.005859375,
|
||
|
|
"rewards/equation_reward_func/std": 0.07639661431312561,
|
||
|
|
"rewards/format_reward_func/mean": 0.9921875,
|
||
|
|
"rewards/format_reward_func/std": 0.08812850713729858,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7774344682693481,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9994958639144897,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5659688115119934,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5751709938049316,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.01716645061969757,
|
||
|
|
"step": 56
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.480924513294465e-05,
|
||
|
|
"clip_ratio/high_mean": 3.480924513294465e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00027445801727784175,
|
||
|
|
"clip_ratio/low_min": 0.00027445801727784175,
|
||
|
|
"clip_ratio/region_mean": 0.0003092672624107864,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 388.0,
|
||
|
|
"completions/max_terminated_length": 388.0,
|
||
|
|
"completions/mean_length": 57.43359375,
|
||
|
|
"completions/mean_terminated_length": 57.43359375,
|
||
|
|
"completions/min_length": 27.0,
|
||
|
|
"completions/min_terminated_length": 27.0,
|
||
|
|
"entropy": 0.5444329058130583,
|
||
|
|
"epoch": 0.015469416785206259,
|
||
|
|
"frac_reward_zero_std": 0.84375,
|
||
|
|
"grad_norm": 0.5821528434753418,
|
||
|
|
"kl": 0.2076412288265096,
|
||
|
|
"learning_rate": 4.880958987534282e-07,
|
||
|
|
"loss": 0.0067,
|
||
|
|
"num_tokens": 5054536.0,
|
||
|
|
"reward": 1.0078125,
|
||
|
|
"reward_std": 0.05973203852772713,
|
||
|
|
"rewards/equation_reward_func/mean": 0.015625,
|
||
|
|
"rewards/equation_reward_func/std": 0.12414088100194931,
|
||
|
|
"rewards/format_reward_func/mean": 0.9921875,
|
||
|
|
"rewards/format_reward_func/std": 0.08812850713729858,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5803041458129883,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9993907809257507,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5755361318588257,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5524532794952393,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.016767196357250214,
|
||
|
|
"step": 58
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00018749161649288403,
|
||
|
|
"clip_ratio/high_mean": 0.00018749161649288403,
|
||
|
|
"clip_ratio/low_mean": 0.0004123110233599113,
|
||
|
|
"clip_ratio/low_min": 0.0004123110233599113,
|
||
|
|
"clip_ratio/region_mean": 0.0005998026398527953,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 108.0,
|
||
|
|
"completions/max_terminated_length": 108.0,
|
||
|
|
"completions/mean_length": 57.244140625,
|
||
|
|
"completions/mean_terminated_length": 57.244140625,
|
||
|
|
"completions/min_length": 23.0,
|
||
|
|
"completions/min_terminated_length": 23.0,
|
||
|
|
"entropy": 0.5230812289648585,
|
||
|
|
"epoch": 0.016002844950213372,
|
||
|
|
"frac_reward_zero_std": 0.890625,
|
||
|
|
"grad_norm": 0.5023806691169739,
|
||
|
|
"kl": 0.11861923730207814,
|
||
|
|
"learning_rate": 4.869727284376277e-07,
|
||
|
|
"loss": -0.0022,
|
||
|
|
"num_tokens": 5161789.0,
|
||
|
|
"reward": 1.001953125,
|
||
|
|
"reward_std": 0.038669899106025696,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0078125,
|
||
|
|
"rewards/equation_reward_func/std": 0.08812850713729858,
|
||
|
|
"rewards/format_reward_func/mean": 0.994140625,
|
||
|
|
"rewards/format_reward_func/std": 0.07639661431312561,
|
||
|
|
"sampling/importance_sampling_ratio/max": 2.0,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000137209892273,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6136685609817505,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.981741189956665,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.015835393220186234,
|
||
|
|
"step": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001258419652003795,
|
||
|
|
"clip_ratio/high_mean": 0.0001258419652003795,
|
||
|
|
"clip_ratio/low_mean": 0.00020840929937548935,
|
||
|
|
"clip_ratio/low_min": 0.00020840929937548935,
|
||
|
|
"clip_ratio/region_mean": 0.00033425126457586884,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 126.0,
|
||
|
|
"completions/max_terminated_length": 126.0,
|
||
|
|
"completions/mean_length": 59.115234375,
|
||
|
|
"completions/mean_terminated_length": 59.115234375,
|
||
|
|
"completions/min_length": 23.0,
|
||
|
|
"completions/min_terminated_length": 23.0,
|
||
|
|
"entropy": 0.5265028360817168,
|
||
|
|
"epoch": 0.016536273115220483,
|
||
|
|
"frac_reward_zero_std": 0.859375,
|
||
|
|
"grad_norm": 0.5980551242828369,
|
||
|
|
"kl": 0.12793309407101738,
|
||
|
|
"learning_rate": 4.858003453744314e-07,
|
||
|
|
"loss": -0.0025,
|
||
|
|
"num_tokens": 5270296.0,
|
||
|
|
"reward": 1.009765625,
|
||
|
|
"reward_std": 0.05681721866130829,
|
||
|
|
"rewards/equation_reward_func/mean": 0.017578125,
|
||
|
|
"rewards/equation_reward_func/std": 0.13154059648513794,
|
||
|
|
"rewards/format_reward_func/mean": 0.9921875,
|
||
|
|
"rewards/format_reward_func/std": 0.08812850713729858,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4510185718536377,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0003494024276733,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6126307845115662,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4899929165840149,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.016140108928084373,
|
||
|
|
"step": 62
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.1965222054471574e-05,
|
||
|
|
"clip_ratio/high_mean": 3.1965222054471574e-05,
|
||
|
|
"clip_ratio/low_mean": 9.001686380037831e-05,
|
||
|
|
"clip_ratio/low_min": 9.001686380037831e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00012198208585484988,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 100.0,
|
||
|
|
"completions/max_terminated_length": 100.0,
|
||
|
|
"completions/mean_length": 58.421875,
|
||
|
|
"completions/mean_terminated_length": 58.421875,
|
||
|
|
"completions/min_length": 25.0,
|
||
|
|
"completions/min_terminated_length": 25.0,
|
||
|
|
"entropy": 0.49459916518794167,
|
||
|
|
"epoch": 0.017069701280227598,
|
||
|
|
"frac_reward_zero_std": 0.796875,
|
||
|
|
"grad_norm": 0.653678297996521,
|
||
|
|
"kl": 0.26154189225700164,
|
||
|
|
"learning_rate": 4.845789930357016e-07,
|
||
|
|
"loss": 0.005,
|
||
|
|
"num_tokens": 5378456.0,
|
||
|
|
"reward": 1.01171875,
|
||
|
|
"reward_std": 0.07352422922849655,
|
||
|
|
"rewards/equation_reward_func/mean": 0.01953125,
|
||
|
|
"rewards/equation_reward_func/std": 0.1385180652141571,
|
||
|
|
"rewards/format_reward_func/mean": 0.9921875,
|
||
|
|
"rewards/format_reward_func/std": 0.08812850713729858,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4458683729171753,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000059723854065,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6116892695426941,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4915308952331543,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.015089752152562141,
|
||
|
|
"step": 64
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 8.667137848937677e-05,
|
||
|
|
"clip_ratio/high_mean": 8.667137848937677e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00017608674049067,
|
||
|
|
"clip_ratio/low_min": 0.00017608674049067,
|
||
|
|
"clip_ratio/region_mean": 0.00026275812221380573,
|
||
|
|
"completions/clipped_ratio": 0.0009765625,
|
||
|
|
"completions/max_length": 200.5,
|
||
|
|
"completions/max_terminated_length": 200.5,
|
||
|
|
"completions/mean_length": 59.0,
|
||
|
|
"completions/mean_terminated_length": 58.88188171386719,
|
||
|
|
"completions/min_length": 25.0,
|
||
|
|
"completions/min_terminated_length": 25.0,
|
||
|
|
"entropy": 0.5012839436531067,
|
||
|
|
"epoch": 0.01760312944523471,
|
||
|
|
"frac_reward_zero_std": 0.78125,
|
||
|
|
"grad_norm": 0.7769564986228943,
|
||
|
|
"kl": 0.14915075670513842,
|
||
|
|
"learning_rate": 4.833089250628786e-07,
|
||
|
|
"loss": -0.0015,
|
||
|
|
"num_tokens": 5594960.0,
|
||
|
|
"reward": 1.0185546875,
|
||
|
|
"reward_std": 0.0828927680850029,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0263671875,
|
||
|
|
"rewards/equation_reward_func/std": 0.16035238653421402,
|
||
|
|
"rewards/format_reward_func/mean": 0.9921875,
|
||
|
|
"rewards/format_reward_func/std": 0.08508053794503212,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5077810287475586,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9981475174427032,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.2692959010847062,
|
||
|
|
"sampling/sampling_logp_difference/max": 12.176925659179688,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0425907289609313,
|
||
|
|
"step": 66
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012458682370682558,
|
||
|
|
"clip_ratio/high_mean": 0.00012458682370682558,
|
||
|
|
"clip_ratio/low_mean": 0.00039236547632349864,
|
||
|
|
"clip_ratio/low_min": 0.00039236547632349864,
|
||
|
|
"clip_ratio/region_mean": 0.0005169522967965653,
|
||
|
|
"completions/clipped_ratio": 0.001953125,
|
||
|
|
"completions/max_length": 1024.0,
|
||
|
|
"completions/max_terminated_length": 173.0,
|
||
|
|
"completions/mean_length": 60.228515625,
|
||
|
|
"completions/mean_terminated_length": 58.342464447021484,
|
||
|
|
"completions/min_length": 30.0,
|
||
|
|
"completions/min_terminated_length": 30.0,
|
||
|
|
"entropy": 0.48701537897189456,
|
||
|
|
"epoch": 0.01813655761024182,
|
||
|
|
"frac_reward_zero_std": 0.796875,
|
||
|
|
"grad_norm": 0.5133281350135803,
|
||
|
|
"kl": 0.14751790546708637,
|
||
|
|
"learning_rate": 4.819904052143058e-07,
|
||
|
|
"loss": 0.0069,
|
||
|
|
"num_tokens": 5703877.0,
|
||
|
|
"reward": 1.015625,
|
||
|
|
"reward_std": 0.08259691298007965,
|
||
|
|
"rewards/equation_reward_func/mean": 0.025390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.15746226906776428,
|
||
|
|
"rewards/format_reward_func/mean": 0.990234375,
|
||
|
|
"rewards/format_reward_func/std": 0.09843364357948303,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4501948356628418,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000070571899414,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5305090546607971,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.633918285369873,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.015203745104372501,
|
||
|
|
"step": 68
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001730913564743888,
|
||
|
|
"clip_ratio/high_mean": 0.0001730913564743888,
|
||
|
|
"clip_ratio/low_mean": 0.00018130017108180455,
|
||
|
|
"clip_ratio/low_min": 0.00018130017108180455,
|
||
|
|
"clip_ratio/region_mean": 0.00035439152755619335,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 101.0,
|
||
|
|
"completions/max_terminated_length": 101.0,
|
||
|
|
"completions/mean_length": 57.494140625,
|
||
|
|
"completions/mean_terminated_length": 57.494140625,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"entropy": 0.499873752395312,
|
||
|
|
"epoch": 0.018669985775248935,
|
||
|
|
"frac_reward_zero_std": 0.8125,
|
||
|
|
"grad_norm": 0.4106377065181732,
|
||
|
|
"kl": 0.14240571359793344,
|
||
|
|
"learning_rate": 4.806237073104548e-07,
|
||
|
|
"loss": 0.0431,
|
||
|
|
"num_tokens": 5811354.0,
|
||
|
|
"reward": 1.0234375,
|
||
|
|
"reward_std": 0.07082759588956833,
|
||
|
|
"rewards/equation_reward_func/mean": 0.025390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.15746226906776428,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4750500917434692,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9996945261955261,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6141188740730286,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4875667095184326,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0145404152572155,
|
||
|
|
"step": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 2.7367269568559197e-05,
|
||
|
|
"clip_ratio/high_mean": 2.7367269568559197e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00012311079060762294,
|
||
|
|
"clip_ratio/low_min": 0.00012311079060762294,
|
||
|
|
"clip_ratio/region_mean": 0.00015047806017618213,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 117.0,
|
||
|
|
"completions/max_terminated_length": 117.0,
|
||
|
|
"completions/mean_length": 56.755859375,
|
||
|
|
"completions/mean_terminated_length": 56.755859375,
|
||
|
|
"completions/min_length": 27.0,
|
||
|
|
"completions/min_terminated_length": 27.0,
|
||
|
|
"entropy": 0.48587797333796817,
|
||
|
|
"epoch": 0.019203413940256046,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.6676278114318848,
|
||
|
|
"kl": 0.13489198436339697,
|
||
|
|
"learning_rate": 4.792091151770602e-07,
|
||
|
|
"loss": -0.0127,
|
||
|
|
"num_tokens": 5918501.0,
|
||
|
|
"reward": 1.0390625,
|
||
|
|
"reward_std": 0.1152973473072052,
|
||
|
|
"rewards/equation_reward_func/mean": 0.044921875,
|
||
|
|
"rewards/equation_reward_func/std": 0.20733514428138733,
|
||
|
|
"rewards/format_reward_func/mean": 0.994140625,
|
||
|
|
"rewards/format_reward_func/std": 0.07639661431312561,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5762015581130981,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0003767013549805,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4934070110321045,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7064208984375,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.014744394458830357,
|
||
|
|
"step": 72
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.112760825600062e-05,
|
||
|
|
"clip_ratio/high_mean": 9.112760825600062e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0004944723299963193,
|
||
|
|
"clip_ratio/low_min": 0.0004944723299963193,
|
||
|
|
"clip_ratio/region_mean": 0.0005855999382523199,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 234.0,
|
||
|
|
"completions/max_terminated_length": 234.0,
|
||
|
|
"completions/mean_length": 57.236328125,
|
||
|
|
"completions/mean_terminated_length": 57.236328125,
|
||
|
|
"completions/min_length": 26.0,
|
||
|
|
"completions/min_terminated_length": 26.0,
|
||
|
|
"entropy": 0.4855133162604438,
|
||
|
|
"epoch": 0.019736842105263157,
|
||
|
|
"frac_reward_zero_std": 0.765625,
|
||
|
|
"grad_norm": 0.6810640692710876,
|
||
|
|
"kl": 0.31850574165582657,
|
||
|
|
"learning_rate": 4.777469225861765e-07,
|
||
|
|
"loss": 0.0063,
|
||
|
|
"num_tokens": 6025774.0,
|
||
|
|
"reward": 1.03125,
|
||
|
|
"reward_std": 0.0962078869342804,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.1939331740140915,
|
||
|
|
"rewards/format_reward_func/mean": 0.9921875,
|
||
|
|
"rewards/format_reward_func/std": 0.08812850713729858,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6487880945205688,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998194575309753,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5815256834030151,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5421000719070435,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.015022373758256435,
|
||
|
|
"step": 74
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.071986035754283e-05,
|
||
|
|
"clip_ratio/high_mean": 9.071986035754283e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0005455782181040073,
|
||
|
|
"clip_ratio/low_min": 0.0005455782181040073,
|
||
|
|
"clip_ratio/region_mean": 0.0006362980784615502,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 168.0,
|
||
|
|
"completions/max_terminated_length": 168.0,
|
||
|
|
"completions/mean_length": 57.16015625,
|
||
|
|
"completions/mean_terminated_length": 57.16015625,
|
||
|
|
"completions/min_length": 28.0,
|
||
|
|
"completions/min_terminated_length": 28.0,
|
||
|
|
"entropy": 0.4848656083146731,
|
||
|
|
"epoch": 0.02027027027027027,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.9179761409759521,
|
||
|
|
"kl": 0.26588521773616475,
|
||
|
|
"learning_rate": 4.762374331951703e-07,
|
||
|
|
"loss": 0.0128,
|
||
|
|
"num_tokens": 6133064.0,
|
||
|
|
"reward": 1.046875,
|
||
|
|
"reward_std": 0.12060342729091644,
|
||
|
|
"rewards/equation_reward_func/mean": 0.05078125,
|
||
|
|
"rewards/equation_reward_func/std": 0.21976542472839355,
|
||
|
|
"rewards/format_reward_func/mean": 0.99609375,
|
||
|
|
"rewards/format_reward_func/std": 0.06243881583213806,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4661178588867188,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9996462464332581,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5549461841583252,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5888841152191162,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.014737057499587536,
|
||
|
|
"step": 76
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.000124341869700907,
|
||
|
|
"clip_ratio/high_mean": 0.000124341869700907,
|
||
|
|
"clip_ratio/low_mean": 0.00036503980906369787,
|
||
|
|
"clip_ratio/low_min": 0.00036503980906369787,
|
||
|
|
"clip_ratio/region_mean": 0.0004893816755308459,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 91.0,
|
||
|
|
"completions/max_terminated_length": 91.0,
|
||
|
|
"completions/mean_length": 56.6640625,
|
||
|
|
"completions/mean_terminated_length": 56.6640625,
|
||
|
|
"completions/min_length": 25.0,
|
||
|
|
"completions/min_terminated_length": 25.0,
|
||
|
|
"entropy": 0.46769310616784626,
|
||
|
|
"epoch": 0.020803698435277383,
|
||
|
|
"frac_reward_zero_std": 0.8125,
|
||
|
|
"grad_norm": 0.49485430121421814,
|
||
|
|
"kl": 0.17351551829940742,
|
||
|
|
"learning_rate": 4.7468096048365814e-07,
|
||
|
|
"loss": 0.0042,
|
||
|
|
"num_tokens": 6240300.0,
|
||
|
|
"reward": 1.033203125,
|
||
|
|
"reward_std": 0.07595369219779968,
|
||
|
|
"rewards/equation_reward_func/mean": 0.03515625,
|
||
|
|
"rewards/equation_reward_func/std": 0.1843547374010086,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.554433822631836,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997377395629883,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6880477070808411,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.44111132621765137,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.014428908936679363,
|
||
|
|
"step": 78
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.466569948320587e-05,
|
||
|
|
"clip_ratio/high_mean": 9.466569948320587e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00037429635995067656,
|
||
|
|
"clip_ratio/low_min": 0.00037429635995067656,
|
||
|
|
"clip_ratio/region_mean": 0.0004689620594338824,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 141.0,
|
||
|
|
"completions/max_terminated_length": 141.0,
|
||
|
|
"completions/mean_length": 55.90625,
|
||
|
|
"completions/mean_terminated_length": 55.90625,
|
||
|
|
"completions/min_length": 22.0,
|
||
|
|
"completions/min_terminated_length": 22.0,
|
||
|
|
"entropy": 0.4568544675906499,
|
||
|
|
"epoch": 0.021337126600284494,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.8507832288742065,
|
||
|
|
"kl": 0.2776802186336782,
|
||
|
|
"learning_rate": 4.730778276884061e-07,
|
||
|
|
"loss": 0.0046,
|
||
|
|
"num_tokens": 6347020.0,
|
||
|
|
"reward": 1.056640625,
|
||
|
|
"reward_std": 0.13053178787231445,
|
||
|
|
"rewards/equation_reward_func/mean": 0.060546875,
|
||
|
|
"rewards/equation_reward_func/std": 0.2387305200099945,
|
||
|
|
"rewards/format_reward_func/mean": 0.99609375,
|
||
|
|
"rewards/format_reward_func/std": 0.06243881583213806,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.431517243385315,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997592568397522,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6875678896903992,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.37459468841552734,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.01453035231679678,
|
||
|
|
"step": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.149079288459486e-05,
|
||
|
|
"clip_ratio/high_mean": 6.149079288459486e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00043590317040474876,
|
||
|
|
"clip_ratio/low_min": 0.00043590317040474876,
|
||
|
|
"clip_ratio/region_mean": 0.0004973939697568616,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 119.5,
|
||
|
|
"completions/max_terminated_length": 119.5,
|
||
|
|
"completions/mean_length": 54.400390625,
|
||
|
|
"completions/mean_terminated_length": 54.400390625,
|
||
|
|
"completions/min_length": 28.0,
|
||
|
|
"completions/min_terminated_length": 28.0,
|
||
|
|
"entropy": 0.4379849069648319,
|
||
|
|
"epoch": 0.021870554765291608,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.6990366578102112,
|
||
|
|
"kl": 0.23277462273836136,
|
||
|
|
"learning_rate": 4.7142836773620227e-07,
|
||
|
|
"loss": 0.0063,
|
||
|
|
"num_tokens": 6559030.0,
|
||
|
|
"reward": 1.06640625,
|
||
|
|
"reward_std": 0.134109266102314,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0693359375,
|
||
|
|
"rewards/equation_reward_func/std": 0.2542656809091568,
|
||
|
|
"rewards/format_reward_func/mean": 0.9970703125,
|
||
|
|
"rewards/format_reward_func/std": 0.053316494449973106,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5191975831985474,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000219702720642,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5883514285087585,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.531247615814209,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.013361877296119928,
|
||
|
|
"step": 82
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.224729845093357e-05,
|
||
|
|
"clip_ratio/high_mean": 6.224729845093357e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0009572154344318227,
|
||
|
|
"clip_ratio/low_min": 0.0009572154344318227,
|
||
|
|
"clip_ratio/region_mean": 0.0010194627199477206,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 86.0,
|
||
|
|
"completions/max_terminated_length": 86.0,
|
||
|
|
"completions/mean_length": 53.50390625,
|
||
|
|
"completions/mean_terminated_length": 53.50390625,
|
||
|
|
"completions/min_length": 27.0,
|
||
|
|
"completions/min_terminated_length": 27.0,
|
||
|
|
"entropy": 0.4128892694910367,
|
||
|
|
"epoch": 0.02240398293029872,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.6463724374771118,
|
||
|
|
"kl": 0.19013702952199513,
|
||
|
|
"learning_rate": 4.6973292317471635e-07,
|
||
|
|
"loss": 0.0052,
|
||
|
|
"num_tokens": 6664664.0,
|
||
|
|
"reward": 1.05078125,
|
||
|
|
"reward_std": 0.11812376976013184,
|
||
|
|
"rewards/equation_reward_func/mean": 0.056640625,
|
||
|
|
"rewards/equation_reward_func/std": 0.23138070106506348,
|
||
|
|
"rewards/format_reward_func/mean": 0.994140625,
|
||
|
|
"rewards/format_reward_func/std": 0.07639661431312561,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5252774953842163,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998437762260437,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6066035628318787,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4998798370361328,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.012673393823206425,
|
||
|
|
"step": 84
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001963357240634246,
|
||
|
|
"clip_ratio/high_mean": 0.0001963357240634246,
|
||
|
|
"clip_ratio/low_mean": 0.0004532367456704378,
|
||
|
|
"clip_ratio/low_min": 0.0004532367456704378,
|
||
|
|
"clip_ratio/region_mean": 0.0006495724697338624,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 89.0,
|
||
|
|
"completions/max_terminated_length": 89.0,
|
||
|
|
"completions/mean_length": 53.46484375,
|
||
|
|
"completions/mean_terminated_length": 53.46484375,
|
||
|
|
"completions/min_length": 26.0,
|
||
|
|
"completions/min_terminated_length": 26.0,
|
||
|
|
"entropy": 0.39042702731158996,
|
||
|
|
"epoch": 0.02293741109530583,
|
||
|
|
"frac_reward_zero_std": 0.734375,
|
||
|
|
"grad_norm": 0.731373131275177,
|
||
|
|
"kl": 0.3146343932797511,
|
||
|
|
"learning_rate": 4.679918461013627e-07,
|
||
|
|
"loss": -0.0157,
|
||
|
|
"num_tokens": 6770158.0,
|
||
|
|
"reward": 1.064453125,
|
||
|
|
"reward_std": 0.11969826370477676,
|
||
|
|
"rewards/equation_reward_func/mean": 0.068359375,
|
||
|
|
"rewards/equation_reward_func/std": 0.25260838866233826,
|
||
|
|
"rewards/format_reward_func/mean": 0.99609375,
|
||
|
|
"rewards/format_reward_func/std": 0.06243881583213806,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8587133884429932,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997268319129944,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6095913052558899,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6198844909667969,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.012392060831189156,
|
||
|
|
"step": 86
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.641253841937416e-05,
|
||
|
|
"clip_ratio/high_mean": 6.641253841937416e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0001936533662956208,
|
||
|
|
"clip_ratio/low_min": 0.0001936533662956208,
|
||
|
|
"clip_ratio/region_mean": 0.00026006590471499495,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 91.0,
|
||
|
|
"completions/max_terminated_length": 91.0,
|
||
|
|
"completions/mean_length": 53.654296875,
|
||
|
|
"completions/mean_terminated_length": 53.654296875,
|
||
|
|
"completions/min_length": 26.0,
|
||
|
|
"completions/min_terminated_length": 26.0,
|
||
|
|
"entropy": 0.3859163332316611,
|
||
|
|
"epoch": 0.023470839260312945,
|
||
|
|
"frac_reward_zero_std": 0.609375,
|
||
|
|
"grad_norm": 0.872736394405365,
|
||
|
|
"kl": 0.19867314977778328,
|
||
|
|
"learning_rate": 4.6620549809017885e-07,
|
||
|
|
"loss": 0.0085,
|
||
|
|
"num_tokens": 6875797.0,
|
||
|
|
"reward": 1.0859375,
|
||
|
|
"reward_std": 0.16314628720283508,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0859375,
|
||
|
|
"rewards/equation_reward_func/std": 0.28054583072662354,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.586866021156311,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001603364944458,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6841540336608887,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4617609977722168,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.01188343670219183,
|
||
|
|
"step": 88
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0002847820578608662,
|
||
|
|
"clip_ratio/high_mean": 0.0002847820578608662,
|
||
|
|
"clip_ratio/low_mean": 0.00031860194091374677,
|
||
|
|
"clip_ratio/low_min": 0.00031860194091374677,
|
||
|
|
"clip_ratio/region_mean": 0.0006033839923070951,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 91.0,
|
||
|
|
"completions/max_terminated_length": 91.0,
|
||
|
|
"completions/mean_length": 54.541015625,
|
||
|
|
"completions/mean_terminated_length": 54.541015625,
|
||
|
|
"completions/min_length": 30.0,
|
||
|
|
"completions/min_terminated_length": 30.0,
|
||
|
|
"entropy": 0.38282785564661026,
|
||
|
|
"epoch": 0.024004267425320056,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.8757621049880981,
|
||
|
|
"kl": 0.19160300720896986,
|
||
|
|
"learning_rate": 4.643742501167366e-07,
|
||
|
|
"loss": -0.0029,
|
||
|
|
"num_tokens": 6981970.0,
|
||
|
|
"reward": 1.064453125,
|
||
|
|
"reward_std": 0.10797597467899323,
|
||
|
|
"rewards/equation_reward_func/mean": 0.06640625,
|
||
|
|
"rewards/equation_reward_func/std": 0.2492343932390213,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4009743928909302,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000730752944946,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6123918890953064,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4903829097747803,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.01211240142583847,
|
||
|
|
"step": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.413285478225185e-05,
|
||
|
|
"clip_ratio/high_mean": 6.413285478225185e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00023037084287756847,
|
||
|
|
"clip_ratio/low_min": 0.00023037084287756847,
|
||
|
|
"clip_ratio/region_mean": 0.0002945036976598203,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 81.0,
|
||
|
|
"completions/max_terminated_length": 81.0,
|
||
|
|
"completions/mean_length": 53.505859375,
|
||
|
|
"completions/mean_terminated_length": 53.505859375,
|
||
|
|
"completions/min_length": 29.0,
|
||
|
|
"completions/min_terminated_length": 29.0,
|
||
|
|
"entropy": 0.3810688671138551,
|
||
|
|
"epoch": 0.02453769559032717,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.6786807775497437,
|
||
|
|
"kl": 0.23459011399083668,
|
||
|
|
"learning_rate": 4.624984824811006e-07,
|
||
|
|
"loss": 0.011,
|
||
|
|
"num_tokens": 7087429.0,
|
||
|
|
"reward": 1.06640625,
|
||
|
|
"reward_std": 0.13730725646018982,
|
||
|
|
"rewards/equation_reward_func/mean": 0.068359375,
|
||
|
|
"rewards/equation_reward_func/std": 0.25260838866233826,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.601763367652893,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997592568397522,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4768765866756439,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7404975891113281,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.011574014090001583,
|
||
|
|
"step": 92
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001584748258917696,
|
||
|
|
"clip_ratio/high_mean": 0.0001584748258917696,
|
||
|
|
"clip_ratio/low_mean": 0.00031917635674795345,
|
||
|
|
"clip_ratio/low_min": 0.00031917635674795345,
|
||
|
|
"clip_ratio/region_mean": 0.000477651182639723,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 127.0,
|
||
|
|
"completions/max_terminated_length": 127.0,
|
||
|
|
"completions/mean_length": 55.142578125,
|
||
|
|
"completions/mean_terminated_length": 55.142578125,
|
||
|
|
"completions/min_length": 26.0,
|
||
|
|
"completions/min_terminated_length": 26.0,
|
||
|
|
"entropy": 0.373777824971411,
|
||
|
|
"epoch": 0.025071123755334282,
|
||
|
|
"frac_reward_zero_std": 0.5625,
|
||
|
|
"grad_norm": 0.6518265008926392,
|
||
|
|
"kl": 0.177696091019445,
|
||
|
|
"learning_rate": 4.605785847288502e-07,
|
||
|
|
"loss": -0.0148,
|
||
|
|
"num_tokens": 7193918.0,
|
||
|
|
"reward": 1.0859375,
|
||
|
|
"reward_std": 0.17715543508529663,
|
||
|
|
"rewards/equation_reward_func/mean": 0.0859375,
|
||
|
|
"rewards/equation_reward_func/std": 0.28054583072662354,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.503134846687317,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9995354413986206,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5218640565872192,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6503481864929199,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.011570579372346401,
|
||
|
|
"step": 94
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.217362902644608e-05,
|
||
|
|
"clip_ratio/high_mean": 9.217362902644608e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0005733486096788612,
|
||
|
|
"clip_ratio/low_min": 0.0005733486096788612,
|
||
|
|
"clip_ratio/region_mean": 0.0006655222419390662,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 97.0,
|
||
|
|
"completions/max_terminated_length": 97.0,
|
||
|
|
"completions/mean_length": 54.259765625,
|
||
|
|
"completions/mean_terminated_length": 54.259765625,
|
||
|
|
"completions/min_length": 28.0,
|
||
|
|
"completions/min_terminated_length": 28.0,
|
||
|
|
"entropy": 0.3661065283748839,
|
||
|
|
"epoch": 0.025604551920341393,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 0.7916492819786072,
|
||
|
|
"kl": 0.3318152477343877,
|
||
|
|
"learning_rate": 4.5861495557018206e-07,
|
||
|
|
"loss": 0.0019,
|
||
|
|
"num_tokens": 7299955.0,
|
||
|
|
"reward": 1.072265625,
|
||
|
|
"reward_std": 0.14144614338874817,
|
||
|
|
"rewards/equation_reward_func/mean": 0.07421875,
|
||
|
|
"rewards/equation_reward_func/std": 0.2623828947544098,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4506229162216187,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998356103897095,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6574238538742065,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4194263219833374,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.011709309183061123,
|
||
|
|
"step": 96
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.440817055085467e-05,
|
||
|
|
"clip_ratio/high_mean": 6.440817055085467e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0001222810533363372,
|
||
|
|
"clip_ratio/low_min": 0.0001222810533363372,
|
||
|
|
"clip_ratio/region_mean": 0.00018668922388719188,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 125.0,
|
||
|
|
"completions/max_terminated_length": 125.0,
|
||
|
|
"completions/mean_length": 54.677734375,
|
||
|
|
"completions/mean_terminated_length": 54.677734375,
|
||
|
|
"completions/min_length": 29.5,
|
||
|
|
"completions/min_terminated_length": 29.5,
|
||
|
|
"entropy": 0.342889907459418,
|
||
|
|
"epoch": 0.026137980085348508,
|
||
|
|
"frac_reward_zero_std": 0.6171875,
|
||
|
|
"grad_norm": 0.7152344584465027,
|
||
|
|
"kl": 0.24752955407732063,
|
||
|
|
"learning_rate": 4.566080027971082e-07,
|
||
|
|
"loss": 0.0083,
|
||
|
|
"num_tokens": 7512065.0,
|
||
|
|
"reward": 1.1142578125,
|
||
|
|
"reward_std": 0.17356323450803757,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1171875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3217283636331558,
|
||
|
|
"rewards/format_reward_func/mean": 0.9970703125,
|
||
|
|
"rewards/format_reward_func/std": 0.053316494449973106,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4866527318954468,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000663995742798,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6695625483989716,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.42328375577926636,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.010475398041307926,
|
||
|
|
"step": 98
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00018635201720624335,
|
||
|
|
"clip_ratio/high_mean": 0.00018635201720624335,
|
||
|
|
"clip_ratio/low_mean": 0.00033971545700397756,
|
||
|
|
"clip_ratio/low_min": 0.00033971545700397756,
|
||
|
|
"clip_ratio/region_mean": 0.0005260674742102209,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 80.0,
|
||
|
|
"completions/max_terminated_length": 80.0,
|
||
|
|
"completions/mean_length": 54.22265625,
|
||
|
|
"completions/mean_terminated_length": 54.22265625,
|
||
|
|
"completions/min_length": 30.0,
|
||
|
|
"completions/min_terminated_length": 30.0,
|
||
|
|
"entropy": 0.3336399926079644,
|
||
|
|
"epoch": 0.02667140825035562,
|
||
|
|
"frac_reward_zero_std": 0.578125,
|
||
|
|
"grad_norm": 0.752447247505188,
|
||
|
|
"kl": 0.22766229386130968,
|
||
|
|
"learning_rate": 4.545581431987694e-07,
|
||
|
|
"loss": -0.0057,
|
||
|
|
"num_tokens": 7617915.0,
|
||
|
|
"reward": 1.123046875,
|
||
|
|
"reward_std": 0.182941734790802,
|
||
|
|
"rewards/equation_reward_func/mean": 0.123046875,
|
||
|
|
"rewards/equation_reward_func/std": 0.32881227135658264,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4538755416870117,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000040888786316,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6096076369285583,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.49493980407714844,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.010414124466478825,
|
||
|
|
"step": 100
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.638364685492383e-05,
|
||
|
|
"clip_ratio/high_mean": 9.638364685492383e-05,
|
||
|
|
"clip_ratio/low_mean": 0.000633861927781254,
|
||
|
|
"clip_ratio/low_min": 0.000633861927781254,
|
||
|
|
"clip_ratio/region_mean": 0.0007302455778699368,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 84.0,
|
||
|
|
"completions/max_terminated_length": 84.0,
|
||
|
|
"completions/mean_length": 55.734375,
|
||
|
|
"completions/mean_terminated_length": 55.734375,
|
||
|
|
"completions/min_length": 30.0,
|
||
|
|
"completions/min_terminated_length": 30.0,
|
||
|
|
"entropy": 0.31998403122027713,
|
||
|
|
"epoch": 0.02720483641536273,
|
||
|
|
"frac_reward_zero_std": 0.609375,
|
||
|
|
"grad_norm": 0.8397956490516663,
|
||
|
|
"kl": 0.30809418112039566,
|
||
|
|
"learning_rate": 4.5246580247487933e-07,
|
||
|
|
"loss": -0.0068,
|
||
|
|
"num_tokens": 7724659.0,
|
||
|
|
"reward": 1.134765625,
|
||
|
|
"reward_std": 0.1721571683883667,
|
||
|
|
"rewards/equation_reward_func/mean": 0.134765625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3418070077896118,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.3653279542922974,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000591278076172,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6534257531166077,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.42552638053894043,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.010140168480575085,
|
||
|
|
"step": 102
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.0491523729223343e-05,
|
||
|
|
"clip_ratio/high_mean": 3.0491523729223343e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00024429223978788487,
|
||
|
|
"clip_ratio/low_min": 0.00024429223978788487,
|
||
|
|
"clip_ratio/region_mean": 0.0002747837635171082,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 82.0,
|
||
|
|
"completions/max_terminated_length": 82.0,
|
||
|
|
"completions/mean_length": 53.490234375,
|
||
|
|
"completions/mean_terminated_length": 53.490234375,
|
||
|
|
"completions/min_length": 27.0,
|
||
|
|
"completions/min_terminated_length": 27.0,
|
||
|
|
"entropy": 0.3079003418485324,
|
||
|
|
"epoch": 0.027738264580369845,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 0.663601279258728,
|
||
|
|
"kl": 0.22134189742306867,
|
||
|
|
"learning_rate": 4.5033141514731786e-07,
|
||
|
|
"loss": 0.0185,
|
||
|
|
"num_tokens": 7830126.0,
|
||
|
|
"reward": 1.103515625,
|
||
|
|
"reward_std": 0.14598248898983002,
|
||
|
|
"rewards/equation_reward_func/mean": 0.103515625,
|
||
|
|
"rewards/equation_reward_func/std": 0.30492907762527466,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8041462898254395,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000555515289307,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47461166977882385,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7452583312988281,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.010407421737909317,
|
||
|
|
"step": 104
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012673301454116072,
|
||
|
|
"clip_ratio/high_mean": 0.00012673301454116072,
|
||
|
|
"clip_ratio/low_mean": 0.0003165318339597434,
|
||
|
|
"clip_ratio/low_min": 0.0003165318339597434,
|
||
|
|
"clip_ratio/region_mean": 0.0004432648485009041,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 96.0,
|
||
|
|
"completions/max_terminated_length": 96.0,
|
||
|
|
"completions/mean_length": 54.42578125,
|
||
|
|
"completions/mean_terminated_length": 54.42578125,
|
||
|
|
"completions/min_length": 29.0,
|
||
|
|
"completions/min_terminated_length": 29.0,
|
||
|
|
"entropy": 0.30098287926779854,
|
||
|
|
"epoch": 0.028271692745376956,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.621425211429596,
|
||
|
|
"kl": 0.2817885786708858,
|
||
|
|
"learning_rate": 4.4815542446989373e-07,
|
||
|
|
"loss": -0.0071,
|
||
|
|
"num_tokens": 7936128.0,
|
||
|
|
"reward": 1.109375,
|
||
|
|
"reward_std": 0.14643804728984833,
|
||
|
|
"rewards/equation_reward_func/mean": 0.111328125,
|
||
|
|
"rewards/equation_reward_func/std": 0.31484565138816833,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.9009522199630737,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000678300857544,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.606683075428009,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6423549652099609,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.009736377745866776,
|
||
|
|
"step": 106
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.985202794066734e-05,
|
||
|
|
"clip_ratio/high_mean": 9.985202794066734e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0005190472616555376,
|
||
|
|
"clip_ratio/low_min": 0.0005190472616555376,
|
||
|
|
"clip_ratio/region_mean": 0.0006188992928299638,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 77.0,
|
||
|
|
"completions/max_terminated_length": 77.0,
|
||
|
|
"completions/mean_length": 53.552734375,
|
||
|
|
"completions/mean_terminated_length": 53.552734375,
|
||
|
|
"completions/min_length": 31.0,
|
||
|
|
"completions/min_terminated_length": 31.0,
|
||
|
|
"entropy": 0.2843737068275611,
|
||
|
|
"epoch": 0.028805120910384067,
|
||
|
|
"frac_reward_zero_std": 0.546875,
|
||
|
|
"grad_norm": 0.629451334476471,
|
||
|
|
"kl": 0.21802533811165226,
|
||
|
|
"learning_rate": 4.4593828233629214e-07,
|
||
|
|
"loss": 0.0029,
|
||
|
|
"num_tokens": 8041515.0,
|
||
|
|
"reward": 1.150390625,
|
||
|
|
"reward_std": 0.18977847695350647,
|
||
|
|
"rewards/equation_reward_func/mean": 0.150390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.35780346393585205,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5413179397583008,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999287724494934,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6856960654258728,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.43263787031173706,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.009411610662937164,
|
||
|
|
"step": 108
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.729989804120528e-05,
|
||
|
|
"clip_ratio/high_mean": 6.729989804120528e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0003927987385799901,
|
||
|
|
"clip_ratio/low_min": 0.0003927987385799901,
|
||
|
|
"clip_ratio/region_mean": 0.0004600986398549543,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 83.0,
|
||
|
|
"completions/max_terminated_length": 83.0,
|
||
|
|
"completions/mean_length": 52.8359375,
|
||
|
|
"completions/mean_terminated_length": 52.8359375,
|
||
|
|
"completions/min_length": 24.0,
|
||
|
|
"completions/min_terminated_length": 24.0,
|
||
|
|
"entropy": 0.2845468541814221,
|
||
|
|
"epoch": 0.02933854907539118,
|
||
|
|
"frac_reward_zero_std": 0.5625,
|
||
|
|
"grad_norm": 0.697964608669281,
|
||
|
|
"kl": 0.24136662897136477,
|
||
|
|
"learning_rate": 4.4368044918622893e-07,
|
||
|
|
"loss": 0.0022,
|
||
|
|
"num_tokens": 8146455.0,
|
||
|
|
"reward": 1.130859375,
|
||
|
|
"reward_std": 0.18825891613960266,
|
||
|
|
"rewards/equation_reward_func/mean": 0.130859375,
|
||
|
|
"rewards/equation_reward_func/std": 0.33757632970809937,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4358296394348145,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999491572380066,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.60647052526474,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5000991821289062,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.009553857147693634,
|
||
|
|
"step": 110
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00022822847434629998,
|
||
|
|
"clip_ratio/high_mean": 0.00022822847434629998,
|
||
|
|
"clip_ratio/low_mean": 0.0002550547134079453,
|
||
|
|
"clip_ratio/low_min": 0.0002550547134079453,
|
||
|
|
"clip_ratio/region_mean": 0.00048328318775424524,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 91.0,
|
||
|
|
"completions/max_terminated_length": 91.0,
|
||
|
|
"completions/mean_length": 54.2265625,
|
||
|
|
"completions/mean_terminated_length": 54.2265625,
|
||
|
|
"completions/min_length": 27.0,
|
||
|
|
"completions/min_terminated_length": 27.0,
|
||
|
|
"entropy": 0.27530702037943733,
|
||
|
|
"epoch": 0.029871977240398292,
|
||
|
|
"frac_reward_zero_std": 0.578125,
|
||
|
|
"grad_norm": 0.7769265174865723,
|
||
|
|
"kl": 0.22397846666475138,
|
||
|
|
"learning_rate": 4.4138239390983e-07,
|
||
|
|
"loss": -0.0023,
|
||
|
|
"num_tokens": 8252491.0,
|
||
|
|
"reward": 1.099609375,
|
||
|
|
"reward_std": 0.17675849795341492,
|
||
|
|
"rewards/equation_reward_func/mean": 0.10546875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3074568510055542,
|
||
|
|
"rewards/format_reward_func/mean": 0.994140625,
|
||
|
|
"rewards/format_reward_func/std": 0.07639661431312561,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4896862506866455,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999532103538513,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6068167686462402,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4995284080505371,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.008937017060816288,
|
||
|
|
"step": 112
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.2603026031413014e-05,
|
||
|
|
"clip_ratio/high_mean": 3.2603026031413014e-05,
|
||
|
|
"clip_ratio/low_mean": 6.739244822205769e-05,
|
||
|
|
"clip_ratio/low_min": 6.739244822205769e-05,
|
||
|
|
"clip_ratio/region_mean": 9.99954742534707e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 86.0,
|
||
|
|
"completions/max_terminated_length": 86.0,
|
||
|
|
"completions/mean_length": 52.9208984375,
|
||
|
|
"completions/mean_terminated_length": 52.9208984375,
|
||
|
|
"completions/min_length": 24.5,
|
||
|
|
"completions/min_terminated_length": 24.5,
|
||
|
|
"entropy": 0.2652134576605426,
|
||
|
|
"epoch": 0.030405405405405407,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.6283072233200073,
|
||
|
|
"kl": 0.3024534620344639,
|
||
|
|
"learning_rate": 4.390445937502557e-07,
|
||
|
|
"loss": -0.0049,
|
||
|
|
"num_tokens": 8463026.0,
|
||
|
|
"reward": 1.119140625,
|
||
|
|
"reward_std": 0.14075149968266487,
|
||
|
|
"rewards/equation_reward_func/mean": 0.12109375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3256715089082718,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.03121940791606903,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.754223883152008,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0002116560935974,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5659388899803162,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6350482702255249,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.008850776590406895,
|
||
|
|
"step": 114
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001623934529359556,
|
||
|
|
"clip_ratio/high_mean": 0.0001623934529359556,
|
||
|
|
"clip_ratio/low_mean": 0.00040348145153580443,
|
||
|
|
"clip_ratio/low_min": 0.00040348145153580443,
|
||
|
|
"clip_ratio/region_mean": 0.00056587490447176,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 85.0,
|
||
|
|
"completions/max_terminated_length": 85.0,
|
||
|
|
"completions/mean_length": 52.896484375,
|
||
|
|
"completions/mean_terminated_length": 52.896484375,
|
||
|
|
"completions/min_length": 27.0,
|
||
|
|
"completions/min_terminated_length": 27.0,
|
||
|
|
"entropy": 0.25305818766355515,
|
||
|
|
"epoch": 0.030938833570412518,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.6711949110031128,
|
||
|
|
"kl": 0.22686809280680287,
|
||
|
|
"learning_rate": 4.3666753420459023e-07,
|
||
|
|
"loss": 0.0053,
|
||
|
|
"num_tokens": 8568341.0,
|
||
|
|
"reward": 1.103515625,
|
||
|
|
"reward_std": 0.12191149592399597,
|
||
|
|
"rewards/equation_reward_func/mean": 0.10546875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3074568510055542,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4110792875289917,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000543594360352,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5361621379852295,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6233186721801758,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.008643017150461674,
|
||
|
|
"step": 116
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.980153926234279e-05,
|
||
|
|
"clip_ratio/high_mean": 9.980153926234279e-05,
|
||
|
|
"clip_ratio/low_mean": 6.652296805340383e-05,
|
||
|
|
"clip_ratio/low_min": 6.652296805340383e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00016632450731574662,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 82.0,
|
||
|
|
"completions/max_terminated_length": 82.0,
|
||
|
|
"completions/mean_length": 52.845703125,
|
||
|
|
"completions/mean_terminated_length": 52.845703125,
|
||
|
|
"completions/min_length": 28.0,
|
||
|
|
"completions/min_terminated_length": 28.0,
|
||
|
|
"entropy": 0.2411729163593716,
|
||
|
|
"epoch": 0.03147226173541963,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.7250849008560181,
|
||
|
|
"kl": 0.22534720102945963,
|
||
|
|
"learning_rate": 4.3425170892301764e-07,
|
||
|
|
"loss": 0.0033,
|
||
|
|
"num_tokens": 8673494.0,
|
||
|
|
"reward": 1.09765625,
|
||
|
|
"reward_std": 0.12994009256362915,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1015625,
|
||
|
|
"rewards/equation_reward_func/std": 0.30236753821372986,
|
||
|
|
"rewards/format_reward_func/mean": 0.99609375,
|
||
|
|
"rewards/format_reward_func/std": 0.06243881583213806,
|
||
|
|
"sampling/importance_sampling_ratio/max": 2.0,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001612901687622,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.670235276222229,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8719024062156677,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.008650041185319424,
|
||
|
|
"step": 118
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.200204810127616e-05,
|
||
|
|
"clip_ratio/high_mean": 3.200204810127616e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00013157916772696708,
|
||
|
|
"clip_ratio/low_min": 0.00013157916772696708,
|
||
|
|
"clip_ratio/region_mean": 0.00016358121582824324,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 85.0,
|
||
|
|
"completions/max_terminated_length": 85.0,
|
||
|
|
"completions/mean_length": 52.23046875,
|
||
|
|
"completions/mean_terminated_length": 52.23046875,
|
||
|
|
"completions/min_length": 30.0,
|
||
|
|
"completions/min_terminated_length": 30.0,
|
||
|
|
"entropy": 0.23818034513129127,
|
||
|
|
"epoch": 0.032005689900426744,
|
||
|
|
"frac_reward_zero_std": 0.609375,
|
||
|
|
"grad_norm": 0.626802384853363,
|
||
|
|
"kl": 0.20635693561699656,
|
||
|
|
"learning_rate": 4.3179761960630357e-07,
|
||
|
|
"loss": -0.0024,
|
||
|
|
"num_tokens": 8778276.0,
|
||
|
|
"reward": 1.142578125,
|
||
|
|
"reward_std": 0.17195375263690948,
|
||
|
|
"rewards/equation_reward_func/mean": 0.142578125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3499840497970581,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.3508808612823486,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0002026557922363,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5984481573104858,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5134154558181763,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.008179975673556328,
|
||
|
|
"step": 120
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001628922182135284,
|
||
|
|
"clip_ratio/high_mean": 0.0001628922182135284,
|
||
|
|
"clip_ratio/low_mean": 0.00015749750632999672,
|
||
|
|
"clip_ratio/low_min": 0.00015749750632999672,
|
||
|
|
"clip_ratio/region_mean": 0.00032038972454352514,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 98.0,
|
||
|
|
"completions/max_terminated_length": 98.0,
|
||
|
|
"completions/mean_length": 53.482421875,
|
||
|
|
"completions/mean_terminated_length": 53.482421875,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"entropy": 0.2308905753824446,
|
||
|
|
"epoch": 0.03253911806543385,
|
||
|
|
"frac_reward_zero_std": 0.765625,
|
||
|
|
"grad_norm": 0.540390133857727,
|
||
|
|
"kl": 0.2013553927342097,
|
||
|
|
"learning_rate": 4.293057759016063e-07,
|
||
|
|
"loss": -0.0017,
|
||
|
|
"num_tokens": 8883955.0,
|
||
|
|
"reward": 1.095703125,
|
||
|
|
"reward_std": 0.10389640927314758,
|
||
|
|
"rewards/equation_reward_func/mean": 0.095703125,
|
||
|
|
"rewards/equation_reward_func/std": 0.2944713830947876,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.3597885370254517,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998118877410889,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5017064213752747,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6897401809692383,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.008392934687435627,
|
||
|
|
"step": 122
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00029407673799950216,
|
||
|
|
"clip_ratio/high_mean": 0.00029407673799950216,
|
||
|
|
"clip_ratio/low_mean": 0.00013097736842205955,
|
||
|
|
"clip_ratio/low_min": 0.00013097736842205955,
|
||
|
|
"clip_ratio/region_mean": 0.0004250541064215617,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 53.802734375,
|
||
|
|
"completions/mean_terminated_length": 53.802734375,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"entropy": 0.2228163286215729,
|
||
|
|
"epoch": 0.033072546230440966,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 0.571448802947998,
|
||
|
|
"kl": 0.2501193769276142,
|
||
|
|
"learning_rate": 4.2677669529663686e-07,
|
||
|
|
"loss": -0.0009,
|
||
|
|
"num_tokens": 8989758.0,
|
||
|
|
"reward": 1.134765625,
|
||
|
|
"reward_std": 0.1630825400352478,
|
||
|
|
"rewards/equation_reward_func/mean": 0.134765625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3418070077896118,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4524681568145752,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000288963317871,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6752034425735474,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.39274129271507263,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.00822390615940094,
|
||
|
|
"step": 124
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.728386066854e-05,
|
||
|
|
"clip_ratio/high_mean": 9.728386066854e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00012625338988275162,
|
||
|
|
"clip_ratio/low_min": 0.00012625338988275162,
|
||
|
|
"clip_ratio/region_mean": 0.00022353725055129163,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 80.0,
|
||
|
|
"completions/max_terminated_length": 80.0,
|
||
|
|
"completions/mean_length": 53.658203125,
|
||
|
|
"completions/mean_terminated_length": 53.658203125,
|
||
|
|
"completions/min_length": 31.0,
|
||
|
|
"completions/min_terminated_length": 31.0,
|
||
|
|
"entropy": 0.21811328331629434,
|
||
|
|
"epoch": 0.03360597439544808,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 0.6349706649780273,
|
||
|
|
"kl": 0.2652018351687325,
|
||
|
|
"learning_rate": 4.2421090301219077e-07,
|
||
|
|
"loss": -0.0099,
|
||
|
|
"num_tokens": 9095479.0,
|
||
|
|
"reward": 1.103515625,
|
||
|
|
"reward_std": 0.15132677555084229,
|
||
|
|
"rewards/equation_reward_func/mean": 0.10546875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3074568510055542,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.3080748319625854,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998948574066162,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5621025562286377,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.57607102394104,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007956698536872864,
|
||
|
|
"step": 126
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 9.971219373659955e-05,
|
||
|
|
"clip_ratio/low_min": 9.971219373659955e-05,
|
||
|
|
"clip_ratio/region_mean": 9.971219373659955e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 249.0,
|
||
|
|
"completions/max_terminated_length": 249.0,
|
||
|
|
"completions/mean_length": 53.609375,
|
||
|
|
"completions/mean_terminated_length": 53.609375,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"entropy": 0.20628702764709791,
|
||
|
|
"epoch": 0.034139402560455195,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.5552213191986084,
|
||
|
|
"kl": 0.19269147598081166,
|
||
|
|
"learning_rate": 4.216089318930741e-07,
|
||
|
|
"loss": 0.0147,
|
||
|
|
"num_tokens": 9201079.0,
|
||
|
|
"reward": 1.09765625,
|
||
|
|
"reward_std": 0.14117899537086487,
|
||
|
|
"rewards/equation_reward_func/mean": 0.099609375,
|
||
|
|
"rewards/equation_reward_func/std": 0.29977133870124817,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4223767518997192,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998998641967773,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4869205355644226,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7196543216705322,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007629196625202894,
|
||
|
|
"step": 128
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.00010101531127778192,
|
||
|
|
"clip_ratio/low_min": 0.00010101531127778192,
|
||
|
|
"clip_ratio/region_mean": 0.00010101531127778192,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 85.5,
|
||
|
|
"completions/max_terminated_length": 85.5,
|
||
|
|
"completions/mean_length": 53.05859375,
|
||
|
|
"completions/mean_terminated_length": 53.05859375,
|
||
|
|
"completions/min_length": 31.0,
|
||
|
|
"completions/min_terminated_length": 31.0,
|
||
|
|
"entropy": 0.19540323979324764,
|
||
|
|
"epoch": 0.0346728307254623,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 0.5377609133720398,
|
||
|
|
"kl": 0.3826492495006985,
|
||
|
|
"learning_rate": 4.189713222974466e-07,
|
||
|
|
"loss": 0.0021,
|
||
|
|
"num_tokens": 9411475.0,
|
||
|
|
"reward": 1.1396484375,
|
||
|
|
"reward_std": 0.15932666510343552,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1396484375,
|
||
|
|
"rewards/equation_reward_func/std": 0.34543485939502716,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6392115950584412,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000347197055817,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5813578218221664,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7083454132080078,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007557510165497661,
|
||
|
|
"step": 130
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.43847990129143e-05,
|
||
|
|
"clip_ratio/high_mean": 9.43847990129143e-05,
|
||
|
|
"clip_ratio/low_mean": 9.666985392363535e-05,
|
||
|
|
"clip_ratio/low_min": 9.666985392363535e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00019105465293654965,
|
||
|
|
"completions/clipped_ratio": 0.001953125,
|
||
|
|
"completions/max_length": 77.0,
|
||
|
|
"completions/max_terminated_length": 77.0,
|
||
|
|
"completions/mean_length": 51.697265625,
|
||
|
|
"completions/mean_terminated_length": 51.64775085449219,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"entropy": 0.19397432108720145,
|
||
|
|
"epoch": 0.03520625889046942,
|
||
|
|
"frac_reward_zero_std": 0.59375,
|
||
|
|
"grad_norm": 0.5314783453941345,
|
||
|
|
"kl": 0.24874685290786955,
|
||
|
|
"learning_rate": 4.162986219846037e-07,
|
||
|
|
"loss": -0.0036,
|
||
|
|
"num_tokens": 9515800.0,
|
||
|
|
"reward": 1.185546875,
|
||
|
|
"reward_std": 0.17774078249931335,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1875,
|
||
|
|
"rewards/equation_reward_func/std": 0.39069411158561707,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5407081842422485,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9988886117935181,
|
||
|
|
"sampling/importance_sampling_ratio/min": 5.010540246597195e-11,
|
||
|
|
"sampling/sampling_logp_difference/max": 23.71689224243164,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.028378766030073166,
|
||
|
|
"step": 132
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00010051935613672767,
|
||
|
|
"clip_ratio/high_mean": 0.00010051935613672767,
|
||
|
|
"clip_ratio/low_mean": 0.00016403257419976094,
|
||
|
|
"clip_ratio/low_min": 0.00016403257419976094,
|
||
|
|
"clip_ratio/region_mean": 0.00026455193033648864,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 87.0,
|
||
|
|
"completions/max_terminated_length": 87.0,
|
||
|
|
"completions/mean_length": 52.8671875,
|
||
|
|
"completions/mean_terminated_length": 52.8671875,
|
||
|
|
"completions/min_length": 27.0,
|
||
|
|
"completions/min_terminated_length": 27.0,
|
||
|
|
"entropy": 0.19188633354173767,
|
||
|
|
"epoch": 0.03573968705547653,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.542492687702179,
|
||
|
|
"kl": 0.24353267459405792,
|
||
|
|
"learning_rate": 4.135913860012219e-07,
|
||
|
|
"loss": 0.0026,
|
||
|
|
"num_tokens": 9620932.0,
|
||
|
|
"reward": 1.10546875,
|
||
|
|
"reward_std": 0.1255940943956375,
|
||
|
|
"rewards/equation_reward_func/mean": 0.10546875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3074568510055542,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.3877536058425903,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999425411224365,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4762346148490906,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.741844654083252,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007330980617552996,
|
||
|
|
"step": 134
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.425126649542815e-05,
|
||
|
|
"clip_ratio/high_mean": 3.425126649542815e-05,
|
||
|
|
"clip_ratio/low_mean": 9.994629848127563e-05,
|
||
|
|
"clip_ratio/low_min": 9.994629848127563e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00013419756497670378,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 99.0,
|
||
|
|
"completions/max_terminated_length": 99.0,
|
||
|
|
"completions/mean_length": 52.87109375,
|
||
|
|
"completions/mean_terminated_length": 52.87109375,
|
||
|
|
"completions/min_length": 25.0,
|
||
|
|
"completions/min_terminated_length": 25.0,
|
||
|
|
"entropy": 0.18152287602424622,
|
||
|
|
"epoch": 0.03627311522048364,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.7582495212554932,
|
||
|
|
"kl": 0.5030062202778127,
|
||
|
|
"learning_rate": 4.10850176566091e-07,
|
||
|
|
"loss": -0.0083,
|
||
|
|
"num_tokens": 9726154.0,
|
||
|
|
"reward": 1.109375,
|
||
|
|
"reward_std": 0.13704082369804382,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1171875,
|
||
|
|
"rewards/equation_reward_func/std": 0.32195815443992615,
|
||
|
|
"rewards/format_reward_func/mean": 0.9921875,
|
||
|
|
"rewards/format_reward_func/std": 0.08812850713729858,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4538846015930176,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0002654790878296,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.39450007677078247,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.93013596534729,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0068960110656917095,
|
||
|
|
"step": 136
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012915206025354564,
|
||
|
|
"clip_ratio/high_mean": 0.00012915206025354564,
|
||
|
|
"clip_ratio/low_mean": 0.00026444589861461683,
|
||
|
|
"clip_ratio/low_min": 0.00026444589861461683,
|
||
|
|
"clip_ratio/region_mean": 0.0003935979588681625,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 87.0,
|
||
|
|
"completions/max_terminated_length": 87.0,
|
||
|
|
"completions/mean_length": 53.25,
|
||
|
|
"completions/mean_terminated_length": 53.25,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"entropy": 0.17901599779725075,
|
||
|
|
"epoch": 0.036806543385490754,
|
||
|
|
"frac_reward_zero_std": 0.59375,
|
||
|
|
"grad_norm": 0.7346581816673279,
|
||
|
|
"kl": 0.20770272612571716,
|
||
|
|
"learning_rate": 4.080755629533566e-07,
|
||
|
|
"loss": 0.0198,
|
||
|
|
"num_tokens": 9831426.0,
|
||
|
|
"reward": 1.150390625,
|
||
|
|
"reward_std": 0.1896313577890396,
|
||
|
|
"rewards/equation_reward_func/mean": 0.16015625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3671095669269562,
|
||
|
|
"rewards/format_reward_func/mean": 0.990234375,
|
||
|
|
"rewards/format_reward_func/std": 0.09843364357948303,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4474687576293945,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.00001859664917,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6884425282478333,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.3733234405517578,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006761064287275076,
|
||
|
|
"step": 138
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00010025367050224708,
|
||
|
|
"clip_ratio/high_mean": 0.00010025367050224708,
|
||
|
|
"clip_ratio/low_mean": 0.0001902976784751647,
|
||
|
|
"clip_ratio/low_min": 0.0001902976784751647,
|
||
|
|
"clip_ratio/region_mean": 0.0002905513489774118,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 78.0,
|
||
|
|
"completions/max_terminated_length": 78.0,
|
||
|
|
"completions/mean_length": 53.2734375,
|
||
|
|
"completions/mean_terminated_length": 53.2734375,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"entropy": 0.18027027406626278,
|
||
|
|
"epoch": 0.03733997155049787,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.530332624912262,
|
||
|
|
"kl": 0.2300557821161217,
|
||
|
|
"learning_rate": 4.052681213742971e-07,
|
||
|
|
"loss": -0.012,
|
||
|
|
"num_tokens": 9936646.0,
|
||
|
|
"reward": 1.099609375,
|
||
|
|
"reward_std": 0.11889472603797913,
|
||
|
|
"rewards/equation_reward_func/mean": 0.103515625,
|
||
|
|
"rewards/equation_reward_func/std": 0.30492907762527466,
|
||
|
|
"rewards/format_reward_func/mean": 0.99609375,
|
||
|
|
"rewards/format_reward_func/std": 0.06243881583213806,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6482826471328735,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998129606246948,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4797065258026123,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7345807552337646,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006873577367514372,
|
||
|
|
"step": 140
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.44635519064549e-05,
|
||
|
|
"clip_ratio/high_mean": 6.44635519064549e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0001672606675937358,
|
||
|
|
"clip_ratio/low_min": 0.0001672606675937358,
|
||
|
|
"clip_ratio/region_mean": 0.0002317242195001907,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 77.0,
|
||
|
|
"completions/max_terminated_length": 77.0,
|
||
|
|
"completions/mean_length": 52.615234375,
|
||
|
|
"completions/mean_terminated_length": 52.615234375,
|
||
|
|
"completions/min_length": 31.0,
|
||
|
|
"completions/min_terminated_length": 31.0,
|
||
|
|
"entropy": 0.16615521907806396,
|
||
|
|
"epoch": 0.03787339971550498,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.6144423484802246,
|
||
|
|
"kl": 0.22150362903873125,
|
||
|
|
"learning_rate": 4.024284348576611e-07,
|
||
|
|
"loss": 0.0031,
|
||
|
|
"num_tokens": 10041545.0,
|
||
|
|
"reward": 1.12109375,
|
||
|
|
"reward_std": 0.13269482553005219,
|
||
|
|
"rewards/equation_reward_func/mean": 0.12109375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3265552520751953,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4515355825424194,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999923706054688,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5423057079315186,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6119253635406494,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006486841011792421,
|
||
|
|
"step": 142
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.2150206000854574e-05,
|
||
|
|
"clip_ratio/high_mean": 3.2150206000854574e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0001318561650502185,
|
||
|
|
"clip_ratio/low_min": 0.0001318561650502185,
|
||
|
|
"clip_ratio/region_mean": 0.00016400637105107307,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 150.0,
|
||
|
|
"completions/max_terminated_length": 150.0,
|
||
|
|
"completions/mean_length": 54.080078125,
|
||
|
|
"completions/mean_terminated_length": 54.080078125,
|
||
|
|
"completions/min_length": 31.0,
|
||
|
|
"completions/min_terminated_length": 31.0,
|
||
|
|
"entropy": 0.18016350434886086,
|
||
|
|
"epoch": 0.03840682788051209,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.6359029412269592,
|
||
|
|
"kl": 0.2206818246179157,
|
||
|
|
"learning_rate": 3.9955709312858744e-07,
|
||
|
|
"loss": -0.0016,
|
||
|
|
"num_tokens": 10147442.0,
|
||
|
|
"reward": 1.07421875,
|
||
|
|
"reward_std": 0.1060032919049263,
|
||
|
|
"rewards/equation_reward_func/mean": 0.07421875,
|
||
|
|
"rewards/equation_reward_func/std": 0.2623828947544098,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5840762853622437,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997929930686951,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47793424129486084,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7382822036743164,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007104428485035896,
|
||
|
|
"step": 144
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.125199272193843e-05,
|
||
|
|
"clip_ratio/high_mean": 6.125199272193843e-05,
|
||
|
|
"clip_ratio/low_mean": 6.607002837376462e-05,
|
||
|
|
"clip_ratio/low_min": 6.607002837376462e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00012732202109570304,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 85.5,
|
||
|
|
"completions/max_terminated_length": 85.5,
|
||
|
|
"completions/mean_length": 54.5869140625,
|
||
|
|
"completions/mean_terminated_length": 54.5869140625,
|
||
|
|
"completions/min_length": 33.5,
|
||
|
|
"completions/min_terminated_length": 33.5,
|
||
|
|
"entropy": 0.1835891335374779,
|
||
|
|
"epoch": 0.038940256045519206,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.6189127564430237,
|
||
|
|
"kl": 0.2157546675039662,
|
||
|
|
"learning_rate": 3.9665469248613616e-07,
|
||
|
|
"loss": -0.0033,
|
||
|
|
"num_tokens": 10359667.0,
|
||
|
|
"reward": 1.1298828125,
|
||
|
|
"reward_std": 0.1303960159420967,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1298828125,
|
||
|
|
"rewards/equation_reward_func/std": 0.336248517036438,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.571405291557312,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000739693641663,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6091319620609283,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.49585986137390137,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007004189770668745,
|
||
|
|
"step": 146
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012751224696532718,
|
||
|
|
"clip_ratio/high_mean": 0.00012751224696532718,
|
||
|
|
"clip_ratio/low_mean": 0.0001873280400306814,
|
||
|
|
"clip_ratio/low_min": 0.0001873280400306814,
|
||
|
|
"clip_ratio/region_mean": 0.0003148402869960086,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 143.0,
|
||
|
|
"completions/max_terminated_length": 143.0,
|
||
|
|
"completions/mean_length": 56.111328125,
|
||
|
|
"completions/mean_terminated_length": 56.111328125,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"entropy": 0.18567656063371235,
|
||
|
|
"epoch": 0.039473684210526314,
|
||
|
|
"frac_reward_zero_std": 0.734375,
|
||
|
|
"grad_norm": 0.6717353463172913,
|
||
|
|
"kl": 0.33979356123341453,
|
||
|
|
"learning_rate": 3.9372183567945314e-07,
|
||
|
|
"loss": 0.0086,
|
||
|
|
"num_tokens": 10466764.0,
|
||
|
|
"reward": 1.099609375,
|
||
|
|
"reward_std": 0.12263225764036179,
|
||
|
|
"rewards/equation_reward_func/mean": 0.099609375,
|
||
|
|
"rewards/equation_reward_func/std": 0.29977133870124817,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6148015260696411,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001806020736694,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5403767228126526,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6154887676239014,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006991642992943525,
|
||
|
|
"step": 148
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00015601687805934084,
|
||
|
|
"clip_ratio/high_mean": 0.00015601687805934084,
|
||
|
|
"clip_ratio/low_mean": 9.35077195107523e-05,
|
||
|
|
"clip_ratio/low_min": 9.35077195107523e-05,
|
||
|
|
"clip_ratio/region_mean": 0.0002495245975700931,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 87.0,
|
||
|
|
"completions/max_terminated_length": 87.0,
|
||
|
|
"completions/mean_length": 54.890625,
|
||
|
|
"completions/mean_terminated_length": 54.890625,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"entropy": 0.18657905691199833,
|
||
|
|
"epoch": 0.04000711237553343,
|
||
|
|
"frac_reward_zero_std": 0.578125,
|
||
|
|
"grad_norm": 0.7030015587806702,
|
||
|
|
"kl": 0.21585925627085897,
|
||
|
|
"learning_rate": 3.907591317825956e-07,
|
||
|
|
"loss": -0.0152,
|
||
|
|
"num_tokens": 10572988.0,
|
||
|
|
"reward": 1.16796875,
|
||
|
|
"reward_std": 0.18523895740509033,
|
||
|
|
"rewards/equation_reward_func/mean": 0.169921875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3759314715862274,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6140424013137817,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000890493392944,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5526527166366577,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5930254459381104,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007370211649686098,
|
||
|
|
"step": 150
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.338281026627455e-05,
|
||
|
|
"clip_ratio/high_mean": 6.338281026627455e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00012458148477081623,
|
||
|
|
"clip_ratio/low_min": 0.00012458148477081623,
|
||
|
|
"clip_ratio/region_mean": 0.00018796429503709078,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 85.0,
|
||
|
|
"completions/max_terminated_length": 85.0,
|
||
|
|
"completions/mean_length": 54.830078125,
|
||
|
|
"completions/mean_terminated_length": 54.830078125,
|
||
|
|
"completions/min_length": 33.0,
|
||
|
|
"completions/min_terminated_length": 33.0,
|
||
|
|
"entropy": 0.19338507536384794,
|
||
|
|
"epoch": 0.04054054054054054,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.6828270554542542,
|
||
|
|
"kl": 0.28864950748781365,
|
||
|
|
"learning_rate": 3.877671960680443e-07,
|
||
|
|
"loss": 0.016,
|
||
|
|
"num_tokens": 10679221.0,
|
||
|
|
"reward": 1.154296875,
|
||
|
|
"reward_std": 0.12717056274414062,
|
||
|
|
"rewards/equation_reward_func/mean": 0.154296875,
|
||
|
|
"rewards/equation_reward_func/std": 0.36158639192581177,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4537763595581055,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997088313102722,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.608879029750824,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4961357116699219,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007639226969331503,
|
||
|
|
"step": 152
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001902342935661889,
|
||
|
|
"clip_ratio/high_mean": 0.0001902342935661889,
|
||
|
|
"clip_ratio/low_mean": 0.00021917261053911515,
|
||
|
|
"clip_ratio/low_min": 0.00021917261053911515,
|
||
|
|
"clip_ratio/region_mean": 0.00040940690410530404,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 85.0,
|
||
|
|
"completions/max_terminated_length": 85.0,
|
||
|
|
"completions/mean_length": 54.658203125,
|
||
|
|
"completions/mean_terminated_length": 54.658203125,
|
||
|
|
"completions/min_length": 33.0,
|
||
|
|
"completions/min_terminated_length": 33.0,
|
||
|
|
"entropy": 0.18605521445473036,
|
||
|
|
"epoch": 0.04107396870554765,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.5871764421463013,
|
||
|
|
"kl": 0.2087138651145829,
|
||
|
|
"learning_rate": 3.847466498789282e-07,
|
||
|
|
"loss": -0.0096,
|
||
|
|
"num_tokens": 10785270.0,
|
||
|
|
"reward": 1.11328125,
|
||
|
|
"reward_std": 0.12815652787685394,
|
||
|
|
"rewards/equation_reward_func/mean": 0.11328125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3172462284564972,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4995094537734985,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998587369918823,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6065520644187927,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.49996471405029297,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007526198402047157,
|
||
|
|
"step": 154
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012783622433845367,
|
||
|
|
"clip_ratio/high_mean": 0.00012783622433845367,
|
||
|
|
"clip_ratio/low_mean": 3.211303717560238e-05,
|
||
|
|
"clip_ratio/low_min": 3.211303717560238e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00015994926151405607,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 87.0,
|
||
|
|
"completions/max_terminated_length": 87.0,
|
||
|
|
"completions/mean_length": 55.439453125,
|
||
|
|
"completions/mean_terminated_length": 55.439453125,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.18379438585705227,
|
||
|
|
"epoch": 0.041607396870554765,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.5001620054244995,
|
||
|
|
"kl": 0.21861587092280388,
|
||
|
|
"learning_rate": 3.816981204999882e-07,
|
||
|
|
"loss": -0.0009,
|
||
|
|
"num_tokens": 10891887.0,
|
||
|
|
"reward": 1.119140625,
|
||
|
|
"reward_std": 0.12671181559562683,
|
||
|
|
"rewards/equation_reward_func/mean": 0.119140625,
|
||
|
|
"rewards/equation_reward_func/std": 0.32427072525024414,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8568929433822632,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0003390312194824,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4759238362312317,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.742497444152832,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.00747784785926342,
|
||
|
|
"step": 156
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00038483808101672266,
|
||
|
|
"clip_ratio/high_mean": 0.00038483808101672266,
|
||
|
|
"clip_ratio/low_mean": 0.00026095764931394823,
|
||
|
|
"clip_ratio/low_min": 0.00026095764931394823,
|
||
|
|
"clip_ratio/region_mean": 0.0006457957303306708,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 85.0,
|
||
|
|
"completions/max_terminated_length": 85.0,
|
||
|
|
"completions/mean_length": 53.755859375,
|
||
|
|
"completions/mean_terminated_length": 53.755859375,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.1828179011742274,
|
||
|
|
"epoch": 0.04214082503556188,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.5133856534957886,
|
||
|
|
"kl": 0.3109063622024324,
|
||
|
|
"learning_rate": 3.786222410273078e-07,
|
||
|
|
"loss": -0.0017,
|
||
|
|
"num_tokens": 10997298.0,
|
||
|
|
"reward": 1.208984375,
|
||
|
|
"reward_std": 0.16108018159866333,
|
||
|
|
"rewards/equation_reward_func/mean": 0.2109375,
|
||
|
|
"rewards/equation_reward_func/std": 0.4083731174468994,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.445540189743042,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998924136161804,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4234831929206848,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8592414855957031,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0071702562272548676,
|
||
|
|
"step": 158
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.23872834795879e-05,
|
||
|
|
"clip_ratio/high_mean": 9.23872834795879e-05,
|
||
|
|
"clip_ratio/low_mean": 3.1494079545761146e-05,
|
||
|
|
"clip_ratio/low_min": 3.1494079545761146e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00012388136302534904,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 83.0,
|
||
|
|
"completions/max_terminated_length": 83.0,
|
||
|
|
"completions/mean_length": 54.71875,
|
||
|
|
"completions/mean_terminated_length": 54.71875,
|
||
|
|
"completions/min_length": 41.0,
|
||
|
|
"completions/min_terminated_length": 41.0,
|
||
|
|
"entropy": 0.19189577591088083,
|
||
|
|
"epoch": 0.04267425320056899,
|
||
|
|
"frac_reward_zero_std": 0.78125,
|
||
|
|
"grad_norm": 0.4394693672657013,
|
||
|
|
"kl": 0.1994515239364571,
|
||
|
|
"learning_rate": 3.755196502368361e-07,
|
||
|
|
"loss": 0.0055,
|
||
|
|
"num_tokens": 11103290.0,
|
||
|
|
"reward": 1.140625,
|
||
|
|
"reward_std": 0.09863612055778503,
|
||
|
|
"rewards/equation_reward_func/mean": 0.140625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3479743003845215,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6096035242080688,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000537633895874,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6076365113258362,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.49817848205566406,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.00804964266717434,
|
||
|
|
"step": 160
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012591962068755593,
|
||
|
|
"clip_ratio/high_mean": 0.00012591962068755593,
|
||
|
|
"clip_ratio/low_mean": 9.727939484744436e-05,
|
||
|
|
"clip_ratio/low_min": 9.727939484744436e-05,
|
||
|
|
"clip_ratio/region_mean": 0.0002231990155350003,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 98.5,
|
||
|
|
"completions/max_terminated_length": 98.5,
|
||
|
|
"completions/mean_length": 55.3720703125,
|
||
|
|
"completions/mean_terminated_length": 55.3720703125,
|
||
|
|
"completions/min_length": 33.0,
|
||
|
|
"completions/min_terminated_length": 33.0,
|
||
|
|
"entropy": 0.18139980029728678,
|
||
|
|
"epoch": 0.0432076813655761,
|
||
|
|
"frac_reward_zero_std": 0.625,
|
||
|
|
"grad_norm": 0.4797929525375366,
|
||
|
|
"kl": 0.20843237472905052,
|
||
|
|
"learning_rate": 3.723909924517314e-07,
|
||
|
|
"loss": 0.0063,
|
||
|
|
"num_tokens": 11316079.0,
|
||
|
|
"reward": 1.150390625,
|
||
|
|
"reward_std": 0.1634674072265625,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1513671875,
|
||
|
|
"rewards/equation_reward_func/std": 0.35236628353595734,
|
||
|
|
"rewards/format_reward_func/mean": 0.9990234375,
|
||
|
|
"rewards/format_reward_func/std": 0.022097086533904076,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4947065114974976,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.999882847070694,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47640350461006165,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.741499662399292,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007098018890246749,
|
||
|
|
"step": 162
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001873802587700387,
|
||
|
|
"clip_ratio/high_mean": 0.0001873802587700387,
|
||
|
|
"clip_ratio/low_mean": 0.000373479687065507,
|
||
|
|
"clip_ratio/low_min": 0.000373479687065507,
|
||
|
|
"clip_ratio/region_mean": 0.0005608599458355457,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 79.0,
|
||
|
|
"completions/max_terminated_length": 79.0,
|
||
|
|
"completions/mean_length": 55.49609375,
|
||
|
|
"completions/mean_terminated_length": 55.49609375,
|
||
|
|
"completions/min_length": 43.0,
|
||
|
|
"completions/min_terminated_length": 43.0,
|
||
|
|
"entropy": 0.1870541183484925,
|
||
|
|
"epoch": 0.043741109530583216,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.6516048908233643,
|
||
|
|
"kl": 0.3076294929616981,
|
||
|
|
"learning_rate": 3.692369174085534e-07,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 11422629.0,
|
||
|
|
"reward": 1.173828125,
|
||
|
|
"reward_std": 0.13640210032463074,
|
||
|
|
"rewards/equation_reward_func/mean": 0.17578125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3810062110424042,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6477184295654297,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000516176223755,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5377401113510132,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6203799247741699,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0076889763586223125,
|
||
|
|
"step": 164
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00016062219704811773,
|
||
|
|
"clip_ratio/high_mean": 0.00016062219704811773,
|
||
|
|
"clip_ratio/low_mean": 6.016539151055945e-05,
|
||
|
|
"clip_ratio/low_min": 6.016539151055945e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00022078758855867718,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 80.0,
|
||
|
|
"completions/max_terminated_length": 80.0,
|
||
|
|
"completions/mean_length": 54.98828125,
|
||
|
|
"completions/mean_terminated_length": 54.98828125,
|
||
|
|
"completions/min_length": 43.0,
|
||
|
|
"completions/min_terminated_length": 43.0,
|
||
|
|
"entropy": 0.18302446396814454,
|
||
|
|
"epoch": 0.044274537695590324,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.6432575583457947,
|
||
|
|
"kl": 0.24118708156877094,
|
||
|
|
"learning_rate": 3.6605808012233004e-07,
|
||
|
|
"loss": -0.0004,
|
||
|
|
"num_tokens": 11528847.0,
|
||
|
|
"reward": 1.1640625,
|
||
|
|
"reward_std": 0.11363443732261658,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1640625,
|
||
|
|
"rewards/equation_reward_func/std": 0.37069445848464966,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4534339904785156,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999691247940063,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6069241166114807,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.49935150146484375,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0071168760769069195,
|
||
|
|
"step": 166
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001567405900762727,
|
||
|
|
"clip_ratio/high_mean": 0.0001567405900762727,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0001567405900762727,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 86.0,
|
||
|
|
"completions/max_terminated_length": 86.0,
|
||
|
|
"completions/mean_length": 54.84375,
|
||
|
|
"completions/mean_terminated_length": 54.84375,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"entropy": 0.17809889341394106,
|
||
|
|
"epoch": 0.04480796586059744,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.5978209972381592,
|
||
|
|
"kl": 0.18232033277551332,
|
||
|
|
"learning_rate": 3.628551407505292e-07,
|
||
|
|
"loss": -0.0072,
|
||
|
|
"num_tokens": 11634959.0,
|
||
|
|
"reward": 1.16015625,
|
||
|
|
"reward_std": 0.1343441903591156,
|
||
|
|
"rewards/equation_reward_func/mean": 0.162109375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3689115643501282,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5012110471725464,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001106262207031,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5386102795600891,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6187629699707031,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007097771856933832,
|
||
|
|
"step": 168
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.233734363069137e-05,
|
||
|
|
"clip_ratio/high_mean": 3.233734363069137e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00019210456068524055,
|
||
|
|
"clip_ratio/low_min": 0.00019210456068524055,
|
||
|
|
"clip_ratio/region_mean": 0.00022444190431593193,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 89.0,
|
||
|
|
"completions/max_terminated_length": 89.0,
|
||
|
|
"completions/mean_length": 54.818359375,
|
||
|
|
"completions/mean_terminated_length": 54.818359375,
|
||
|
|
"completions/min_length": 42.0,
|
||
|
|
"completions/min_terminated_length": 42.0,
|
||
|
|
"entropy": 0.1777913181318177,
|
||
|
|
"epoch": 0.04534139402560455,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.5563479661941528,
|
||
|
|
"kl": 0.2434103993905915,
|
||
|
|
"learning_rate": 3.5962876445596224e-07,
|
||
|
|
"loss": 0.0068,
|
||
|
|
"num_tokens": 11741194.0,
|
||
|
|
"reward": 1.10546875,
|
||
|
|
"reward_std": 0.13098736107349396,
|
||
|
|
"rewards/equation_reward_func/mean": 0.10546875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3074568510055542,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5672227144241333,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999104142189026,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5404222011566162,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6154046058654785,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007332408335059881,
|
||
|
|
"step": 170
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00015626028956224522,
|
||
|
|
"clip_ratio/high_mean": 0.00015626028956224522,
|
||
|
|
"clip_ratio/low_mean": 0.0001587454591774278,
|
||
|
|
"clip_ratio/low_min": 0.0001587454591774278,
|
||
|
|
"clip_ratio/region_mean": 0.00031500574873967306,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 87.0,
|
||
|
|
"completions/max_terminated_length": 87.0,
|
||
|
|
"completions/mean_length": 53.61328125,
|
||
|
|
"completions/mean_terminated_length": 53.61328125,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"entropy": 0.1705088723036978,
|
||
|
|
"epoch": 0.04587482219061166,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 0.6292311549186707,
|
||
|
|
"kl": 0.20615031073490778,
|
||
|
|
"learning_rate": 3.563796212686475e-07,
|
||
|
|
"loss": -0.004,
|
||
|
|
"num_tokens": 11846532.0,
|
||
|
|
"reward": 1.142578125,
|
||
|
|
"reward_std": 0.14768996834754944,
|
||
|
|
"rewards/equation_reward_func/mean": 0.142578125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3499840497970581,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6039351224899292,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001249313354492,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6371393799781799,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4724600315093994,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0072227707132697105,
|
||
|
|
"step": 172
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.414075045949883e-05,
|
||
|
|
"clip_ratio/high_mean": 9.414075045949883e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0002569324860814959,
|
||
|
|
"clip_ratio/low_min": 0.0002569324860814959,
|
||
|
|
"clip_ratio/region_mean": 0.0003510732365409947,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 78.0,
|
||
|
|
"completions/max_terminated_length": 78.0,
|
||
|
|
"completions/mean_length": 54.255859375,
|
||
|
|
"completions/mean_terminated_length": 54.255859375,
|
||
|
|
"completions/min_length": 43.0,
|
||
|
|
"completions/min_terminated_length": 43.0,
|
||
|
|
"entropy": 0.17187187944849333,
|
||
|
|
"epoch": 0.046408250355618776,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.5958592891693115,
|
||
|
|
"kl": 0.22016057992974916,
|
||
|
|
"learning_rate": 3.531083859466635e-07,
|
||
|
|
"loss": 0.0044,
|
||
|
|
"num_tokens": 11952343.0,
|
||
|
|
"reward": 1.14453125,
|
||
|
|
"reward_std": 0.1269814372062683,
|
||
|
|
"rewards/equation_reward_func/mean": 0.14453125,
|
||
|
|
"rewards/equation_reward_func/std": 0.35197147727012634,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.427687168121338,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000633001327515,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.609822154045105,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.49458789825439453,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007299026474356651,
|
||
|
|
"step": 174
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001271874352823943,
|
||
|
|
"clip_ratio/high_mean": 0.0001271874352823943,
|
||
|
|
"clip_ratio/low_mean": 0.00016112958676078252,
|
||
|
|
"clip_ratio/low_min": 0.00016112958676078252,
|
||
|
|
"clip_ratio/region_mean": 0.0002883170220431768,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 85.0,
|
||
|
|
"completions/max_terminated_length": 85.0,
|
||
|
|
"completions/mean_length": 54.880859375,
|
||
|
|
"completions/mean_terminated_length": 54.880859375,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.1630988936457369,
|
||
|
|
"epoch": 0.04694167852062589,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.5851055383682251,
|
||
|
|
"kl": 0.4657417970399062,
|
||
|
|
"learning_rate": 3.498157378360204e-07,
|
||
|
|
"loss": -0.0006,
|
||
|
|
"num_tokens": 12058658.0,
|
||
|
|
"reward": 1.119140625,
|
||
|
|
"reward_std": 0.11849410086870193,
|
||
|
|
"rewards/equation_reward_func/mean": 0.12109375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3265552520751953,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4064489603042603,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000070333480835,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5667109489440918,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5679059028625488,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0071370163932442665,
|
||
|
|
"step": 176
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 2.9932950080061953e-05,
|
||
|
|
"clip_ratio/high_mean": 2.9932950080061953e-05,
|
||
|
|
"clip_ratio/low_mean": 3.128128203873833e-05,
|
||
|
|
"clip_ratio/low_min": 3.128128203873833e-05,
|
||
|
|
"clip_ratio/region_mean": 6.121423211880028e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 79.5,
|
||
|
|
"completions/max_terminated_length": 79.5,
|
||
|
|
"completions/mean_length": 54.720703125,
|
||
|
|
"completions/mean_terminated_length": 54.720703125,
|
||
|
|
"completions/min_length": 34.5,
|
||
|
|
"completions/min_terminated_length": 34.5,
|
||
|
|
"entropy": 0.1649447451863024,
|
||
|
|
"epoch": 0.047475106685633005,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.5909639000892639,
|
||
|
|
"kl": 0.45460431194967693,
|
||
|
|
"learning_rate": 3.465023607295784e-07,
|
||
|
|
"loss": 0.0043,
|
||
|
|
"num_tokens": 12270948.0,
|
||
|
|
"reward": 1.107421875,
|
||
|
|
"reward_std": 0.10856572538614273,
|
||
|
|
"rewards/equation_reward_func/mean": 0.107421875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3079848438501358,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.519451916217804,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000729262828827,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4784931391477585,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.745232105255127,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0071322976145893335,
|
||
|
|
"step": 178
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.297189651781486e-05,
|
||
|
|
"clip_ratio/high_mean": 6.297189651781486e-05,
|
||
|
|
"clip_ratio/low_mean": 3.229974148174127e-05,
|
||
|
|
"clip_ratio/low_min": 3.229974148174127e-05,
|
||
|
|
"clip_ratio/region_mean": 9.527163799955613e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 103.0,
|
||
|
|
"completions/max_terminated_length": 103.0,
|
||
|
|
"completions/mean_length": 54.529296875,
|
||
|
|
"completions/mean_terminated_length": 54.529296875,
|
||
|
|
"completions/min_length": 41.0,
|
||
|
|
"completions/min_terminated_length": 41.0,
|
||
|
|
"entropy": 0.15816421620547771,
|
||
|
|
"epoch": 0.04800853485064011,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.6006011962890625,
|
||
|
|
"kl": 0.2346639575229751,
|
||
|
|
"learning_rate": 3.4316894272504225e-07,
|
||
|
|
"loss": -0.0081,
|
||
|
|
"num_tokens": 12377027.0,
|
||
|
|
"reward": 1.130859375,
|
||
|
|
"reward_std": 0.12802553176879883,
|
||
|
|
"rewards/equation_reward_func/mean": 0.130859375,
|
||
|
|
"rewards/equation_reward_func/std": 0.33757632970809937,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8673962354660034,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999069571495056,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4734019935131073,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7478103637695312,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007345444522798061,
|
||
|
|
"step": 180
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.469187145638798e-05,
|
||
|
|
"clip_ratio/high_mean": 6.469187145638798e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0001901625332215594,
|
||
|
|
"clip_ratio/low_min": 0.0001901625332215594,
|
||
|
|
"clip_ratio/region_mean": 0.00025485440467794734,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 80.0,
|
||
|
|
"completions/max_terminated_length": 80.0,
|
||
|
|
"completions/mean_length": 54.880859375,
|
||
|
|
"completions/mean_terminated_length": 54.880859375,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.15701940614316198,
|
||
|
|
"epoch": 0.04854196301564723,
|
||
|
|
"frac_reward_zero_std": 0.796875,
|
||
|
|
"grad_norm": 0.620672881603241,
|
||
|
|
"kl": 0.24056435086660916,
|
||
|
|
"learning_rate": 3.398161760820628e-07,
|
||
|
|
"loss": 0.0146,
|
||
|
|
"num_tokens": 12483326.0,
|
||
|
|
"reward": 1.078125,
|
||
|
|
"reward_std": 0.08633885532617569,
|
||
|
|
"rewards/equation_reward_func/mean": 0.078125,
|
||
|
|
"rewards/equation_reward_func/std": 0.26863065361976624,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4245080947875977,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000905990600586,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.7031283974647522,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.35382652282714844,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007266565226018429,
|
||
|
|
"step": 182
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012663751517215537,
|
||
|
|
"clip_ratio/high_mean": 0.00012663751517215537,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.00012663751517215537,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 83.0,
|
||
|
|
"completions/max_terminated_length": 83.0,
|
||
|
|
"completions/mean_length": 54.255859375,
|
||
|
|
"completions/mean_terminated_length": 54.255859375,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.16044065687391493,
|
||
|
|
"epoch": 0.04907539118065434,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 0.6004108786582947,
|
||
|
|
"kl": 0.18950829137530592,
|
||
|
|
"learning_rate": 3.364447570784731e-07,
|
||
|
|
"loss": -0.0007,
|
||
|
|
"num_tokens": 12589121.0,
|
||
|
|
"reward": 1.103515625,
|
||
|
|
"reward_std": 0.14854492247104645,
|
||
|
|
"rewards/equation_reward_func/mean": 0.10546875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3074568510055542,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.444663643836975,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.00014328956604,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6588608026504517,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.41724300384521484,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007076562847942114,
|
||
|
|
"step": 184
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0002561409103994568,
|
||
|
|
"clip_ratio/high_mean": 0.0002561409103994568,
|
||
|
|
"clip_ratio/low_mean": 0.0002564327636112769,
|
||
|
|
"clip_ratio/low_min": 0.0002564327636112769,
|
||
|
|
"clip_ratio/region_mean": 0.0005125736740107337,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 78.0,
|
||
|
|
"completions/max_terminated_length": 78.0,
|
||
|
|
"completions/mean_length": 54.208984375,
|
||
|
|
"completions/mean_terminated_length": 54.208984375,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.1514270328399208,
|
||
|
|
"epoch": 0.04960881934566145,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 0.45418864488601685,
|
||
|
|
"kl": 0.20794780593779352,
|
||
|
|
"learning_rate": 3.3305538586569116e-07,
|
||
|
|
"loss": 0.0119,
|
||
|
|
"num_tokens": 12694964.0,
|
||
|
|
"reward": 1.18359375,
|
||
|
|
"reward_std": 0.15478995442390442,
|
||
|
|
"rewards/equation_reward_func/mean": 0.18359375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3875311613082886,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6467537879943848,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998853802680969,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.48188647627830505,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7300467491149902,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006952677387744188,
|
||
|
|
"step": 186
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0002880274227613376,
|
||
|
|
"clip_ratio/high_mean": 0.0002880274227613376,
|
||
|
|
"clip_ratio/low_mean": 0.00047546141043615836,
|
||
|
|
"clip_ratio/low_min": 0.00047546141043615836,
|
||
|
|
"clip_ratio/region_mean": 0.000763488836431255,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 79.0,
|
||
|
|
"completions/max_terminated_length": 79.0,
|
||
|
|
"completions/mean_length": 54.818359375,
|
||
|
|
"completions/mean_terminated_length": 54.818359375,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.15331672401063973,
|
||
|
|
"epoch": 0.050142247510668564,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 0.675754725933075,
|
||
|
|
"kl": 0.22493650681442684,
|
||
|
|
"learning_rate": 3.296487663233168e-07,
|
||
|
|
"loss": -0.019,
|
||
|
|
"num_tokens": 12801319.0,
|
||
|
|
"reward": 1.14453125,
|
||
|
|
"reward_std": 0.15150675177574158,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1484375,
|
||
|
|
"rewards/equation_reward_func/std": 0.35588082671165466,
|
||
|
|
"rewards/format_reward_func/mean": 0.99609375,
|
||
|
|
"rewards/format_reward_func/std": 0.06243881583213806,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.644798994064331,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999932646751404,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.3253546357154846,
|
||
|
|
"sampling/sampling_logp_difference/max": 1.1228394508361816,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007331944070756435,
|
||
|
|
"step": 188
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001251859066542238,
|
||
|
|
"clip_ratio/high_mean": 0.0001251859066542238,
|
||
|
|
"clip_ratio/low_mean": 0.0002512690197262499,
|
||
|
|
"clip_ratio/low_min": 0.0002512690197262499,
|
||
|
|
"clip_ratio/region_mean": 0.0003764549263804737,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 109.0,
|
||
|
|
"completions/max_terminated_length": 109.0,
|
||
|
|
"completions/mean_length": 53.7421875,
|
||
|
|
"completions/mean_terminated_length": 53.7421875,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.1532320384350088,
|
||
|
|
"epoch": 0.05067567567567568,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 0.808289647102356,
|
||
|
|
"kl": 0.21676196778813997,
|
||
|
|
"learning_rate": 3.2622560591295606e-07,
|
||
|
|
"loss": 0.009,
|
||
|
|
"num_tokens": 12906755.0,
|
||
|
|
"reward": 1.154296875,
|
||
|
|
"reward_std": 0.1503761112689972,
|
||
|
|
"rewards/equation_reward_func/mean": 0.16015625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3671095669269562,
|
||
|
|
"rewards/format_reward_func/mean": 0.994140625,
|
||
|
|
"rewards/format_reward_func/std": 0.07639661431312561,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5660736560821533,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0004606246948242,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.3685874342918396,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.998077392578125,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0070484536699950695,
|
||
|
|
"step": 190
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012767854301879802,
|
||
|
|
"clip_ratio/high_mean": 0.00012767854301879802,
|
||
|
|
"clip_ratio/low_mean": 0.0001271537104104128,
|
||
|
|
"clip_ratio/low_min": 0.0001271537104104128,
|
||
|
|
"clip_ratio/region_mean": 0.00025483225342921086,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 88.0,
|
||
|
|
"completions/max_terminated_length": 88.0,
|
||
|
|
"completions/mean_length": 54.3984375,
|
||
|
|
"completions/mean_terminated_length": 54.3984375,
|
||
|
|
"completions/min_length": 33.0,
|
||
|
|
"completions/min_terminated_length": 33.0,
|
||
|
|
"entropy": 0.1483436483475897,
|
||
|
|
"epoch": 0.051209103840682786,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.5794490575790405,
|
||
|
|
"kl": 0.2033289468122853,
|
||
|
|
"learning_rate": 3.227866155313002e-07,
|
||
|
|
"loss": -0.0003,
|
||
|
|
"num_tokens": 13012791.0,
|
||
|
|
"reward": 1.15234375,
|
||
|
|
"reward_std": 0.13749760389328003,
|
||
|
|
"rewards/equation_reward_func/mean": 0.15234375,
|
||
|
|
"rewards/equation_reward_func/std": 0.35970520973205566,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.3876280784606934,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999657869338989,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.644438624382019,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.439375638961792,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0071494379080832005,
|
||
|
|
"step": 192
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.00019094683294598426,
|
||
|
|
"clip_ratio/low_min": 0.00019094683294598426,
|
||
|
|
"clip_ratio/region_mean": 0.00019094683294598426,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 86.0,
|
||
|
|
"completions/max_terminated_length": 86.0,
|
||
|
|
"completions/mean_length": 54.8193359375,
|
||
|
|
"completions/mean_terminated_length": 54.8193359375,
|
||
|
|
"completions/min_length": 40.5,
|
||
|
|
"completions/min_terminated_length": 40.5,
|
||
|
|
"entropy": 0.1460409445895089,
|
||
|
|
"epoch": 0.0517425320056899,
|
||
|
|
"frac_reward_zero_std": 0.6953125,
|
||
|
|
"grad_norm": 0.5141434669494629,
|
||
|
|
"kl": 0.20667694177892473,
|
||
|
|
"learning_rate": 3.1933250936249213e-07,
|
||
|
|
"loss": -0.0011,
|
||
|
|
"num_tokens": 13225486.0,
|
||
|
|
"reward": 1.1396484375,
|
||
|
|
"reward_std": 0.1303311251103878,
|
||
|
|
"rewards/equation_reward_func/mean": 0.140625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3479743003845215,
|
||
|
|
"rewards/format_reward_func/mean": 0.9990234375,
|
||
|
|
"rewards/format_reward_func/std": 0.022097086533904076,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4357757568359375,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000481605529785,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6091239154338837,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5029124021530151,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007287219632416964,
|
||
|
|
"step": 194
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.000539941748785269,
|
||
|
|
"clip_ratio/high_mean": 0.000539941748785269,
|
||
|
|
"clip_ratio/low_mean": 0.00046990027314879827,
|
||
|
|
"clip_ratio/low_min": 0.00046990027314879827,
|
||
|
|
"clip_ratio/region_mean": 0.0010098420154665494,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 81.0,
|
||
|
|
"completions/max_terminated_length": 81.0,
|
||
|
|
"completions/mean_length": 53.841796875,
|
||
|
|
"completions/mean_terminated_length": 53.841796875,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.14552338793873787,
|
||
|
|
"epoch": 0.052275960170697015,
|
||
|
|
"frac_reward_zero_std": 0.609375,
|
||
|
|
"grad_norm": 1.226148247718811,
|
||
|
|
"kl": 0.2773982307149304,
|
||
|
|
"learning_rate": 3.158640047298098e-07,
|
||
|
|
"loss": 0.0025,
|
||
|
|
"num_tokens": 13331053.0,
|
||
|
|
"reward": 1.1875,
|
||
|
|
"reward_std": 0.15940433740615845,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1875,
|
||
|
|
"rewards/equation_reward_func/std": 0.39069411158561707,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4396930932998657,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000059962272644,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.621496319770813,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.47562527656555176,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006564574781805277,
|
||
|
|
"step": 196
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0003576159328480975,
|
||
|
|
"clip_ratio/high_mean": 0.0003576159328480975,
|
||
|
|
"clip_ratio/low_mean": 0.000351945691767873,
|
||
|
|
"clip_ratio/low_min": 0.000351945691767873,
|
||
|
|
"clip_ratio/region_mean": 0.0007095616246159705,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 78.0,
|
||
|
|
"completions/max_terminated_length": 78.0,
|
||
|
|
"completions/mean_length": 53.451171875,
|
||
|
|
"completions/mean_terminated_length": 53.451171875,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"entropy": 0.14037801759938398,
|
||
|
|
"epoch": 0.05280938833570412,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.4710004925727844,
|
||
|
|
"kl": 0.24111700699561173,
|
||
|
|
"learning_rate": 3.123818219466981e-07,
|
||
|
|
"loss": 0.0009,
|
||
|
|
"num_tokens": 13436436.0,
|
||
|
|
"reward": 1.212890625,
|
||
|
|
"reward_std": 0.1142803430557251,
|
||
|
|
"rewards/equation_reward_func/mean": 0.212890625,
|
||
|
|
"rewards/equation_reward_func/std": 0.409751296043396,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4310200214385986,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001682043075562,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6036964058876038,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5046838521957397,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.00685081584379077,
|
||
|
|
"step": 198
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0001622872304223064,
|
||
|
|
"clip_ratio/low_min": 0.0001622872304223064,
|
||
|
|
"clip_ratio/region_mean": 0.0001622872304223064,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 53.625,
|
||
|
|
"completions/mean_terminated_length": 53.625,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.13289273199107912,
|
||
|
|
"epoch": 0.05334281650071124,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.6205074787139893,
|
||
|
|
"kl": 0.3644645079556439,
|
||
|
|
"learning_rate": 3.088866841671789e-07,
|
||
|
|
"loss": 0.0018,
|
||
|
|
"num_tokens": 13541980.0,
|
||
|
|
"reward": 1.146484375,
|
||
|
|
"reward_std": 0.13825923204421997,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1484375,
|
||
|
|
"rewards/equation_reward_func/std": 0.35588082671165466,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4064486026763916,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000766515731812,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.607798159122467,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4979124069213867,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006802752614021301,
|
||
|
|
"step": 200
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.000385660015227687,
|
||
|
|
"clip_ratio/low_min": 0.000385660015227687,
|
||
|
|
"clip_ratio/region_mean": 0.000385660015227687,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 79.0,
|
||
|
|
"completions/max_terminated_length": 79.0,
|
||
|
|
"completions/mean_length": 54.3125,
|
||
|
|
"completions/mean_terminated_length": 54.3125,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.13150467558039558,
|
||
|
|
"epoch": 0.05387624466571835,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.44971543550491333,
|
||
|
|
"kl": 0.2712813439882464,
|
||
|
|
"learning_rate": 3.0537931723567253e-07,
|
||
|
|
"loss": -0.002,
|
||
|
|
"num_tokens": 13648084.0,
|
||
|
|
"reward": 1.140625,
|
||
|
|
"reward_std": 0.10882970690727234,
|
||
|
|
"rewards/equation_reward_func/mean": 0.140625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3479743003845215,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5713891983032227,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.999980628490448,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5369875431060791,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6217803955078125,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006598425097763538,
|
||
|
|
"step": 202
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00022619308502827253,
|
||
|
|
"clip_ratio/high_mean": 0.00022619308502827253,
|
||
|
|
"clip_ratio/low_mean": 0.000285603701033526,
|
||
|
|
"clip_ratio/low_min": 0.000285603701033526,
|
||
|
|
"clip_ratio/region_mean": 0.0005117967860617986,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 77.0,
|
||
|
|
"completions/max_terminated_length": 77.0,
|
||
|
|
"completions/mean_length": 54.66796875,
|
||
|
|
"completions/mean_terminated_length": 54.66796875,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.12755409379800162,
|
||
|
|
"epoch": 0.05440967283072546,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.6239610910415649,
|
||
|
|
"kl": 0.2788337940971057,
|
||
|
|
"learning_rate": 3.01860449536259e-07,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"num_tokens": 13754410.0,
|
||
|
|
"reward": 1.1015625,
|
||
|
|
"reward_std": 0.12901148200035095,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1015625,
|
||
|
|
"rewards/equation_reward_func/std": 0.30236753821372986,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8019253015518188,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001003742218018,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.48298075795173645,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.727778434753418,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0065957228653132915,
|
||
|
|
"step": 204
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00016267979258878363,
|
||
|
|
"clip_ratio/high_mean": 0.00016267979258878363,
|
||
|
|
"clip_ratio/low_mean": 0.0004494503470292936,
|
||
|
|
"clip_ratio/low_min": 0.0004494503470292936,
|
||
|
|
"clip_ratio/region_mean": 0.0006121301396180772,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 72.0,
|
||
|
|
"completions/max_terminated_length": 72.0,
|
||
|
|
"completions/mean_length": 52.990234375,
|
||
|
|
"completions/mean_terminated_length": 52.990234375,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.12572518384291065,
|
||
|
|
"epoch": 0.054943100995732574,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 0.7820398211479187,
|
||
|
|
"kl": 0.22174248699512747,
|
||
|
|
"learning_rate": 2.983308118414131e-07,
|
||
|
|
"loss": 0.0092,
|
||
|
|
"num_tokens": 13859429.0,
|
||
|
|
"reward": 1.173828125,
|
||
|
|
"reward_std": 0.14716199040412903,
|
||
|
|
"rewards/equation_reward_func/mean": 0.17578125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3810062110424042,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.473716378211975,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001775026321411,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6095648407936096,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4950098991394043,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006203033495694399,
|
||
|
|
"step": 206
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00019132617227215733,
|
||
|
|
"clip_ratio/high_mean": 0.00019132617227215733,
|
||
|
|
"clip_ratio/low_mean": 0.00019700702978298068,
|
||
|
|
"clip_ratio/low_min": 0.00019700702978298068,
|
||
|
|
"clip_ratio/region_mean": 0.000388333202055138,
|
||
|
|
"completions/clipped_ratio": 0.001953125,
|
||
|
|
"completions/max_length": 75.0,
|
||
|
|
"completions/max_terminated_length": 75.0,
|
||
|
|
"completions/mean_length": 53.828125,
|
||
|
|
"completions/mean_terminated_length": 53.78669357299805,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.12475637284417947,
|
||
|
|
"epoch": 0.05547652916073969,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.7352174520492554,
|
||
|
|
"kl": 0.19599058396286434,
|
||
|
|
"learning_rate": 2.9479113716024275e-07,
|
||
|
|
"loss": -0.0059,
|
||
|
|
"num_tokens": 13965125.0,
|
||
|
|
"reward": 1.14453125,
|
||
|
|
"reward_std": 0.13664263486862183,
|
||
|
|
"rewards/equation_reward_func/mean": 0.146484375,
|
||
|
|
"rewards/equation_reward_func/std": 0.35393697023391724,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.646515965461731,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9995404481887817,
|
||
|
|
"sampling/importance_sampling_ratio/min": 9.77650252420581e-11,
|
||
|
|
"sampling/sampling_logp_difference/max": 23.04845428466797,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.01847986876964569,
|
||
|
|
"step": 208
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.379291673708293e-05,
|
||
|
|
"clip_ratio/high_mean": 3.379291673708293e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00019734271983098652,
|
||
|
|
"clip_ratio/low_min": 0.00019734271983098652,
|
||
|
|
"clip_ratio/region_mean": 0.00023113563656806946,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 53.04296875,
|
||
|
|
"completions/mean_terminated_length": 53.04296875,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.12059136997494432,
|
||
|
|
"epoch": 0.0560099573257468,
|
||
|
|
"frac_reward_zero_std": 0.7265625,
|
||
|
|
"grad_norm": 0.474179208278656,
|
||
|
|
"kl": 0.19687053250769773,
|
||
|
|
"learning_rate": 2.912421605862632e-07,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 14175505.0,
|
||
|
|
"reward": 1.1142578125,
|
||
|
|
"reward_std": 0.11813334748148918,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1142578125,
|
||
|
|
"rewards/equation_reward_func/std": 0.31834255158901215,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4857916235923767,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999574422836304,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.49108970165252686,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7114477157592773,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006181263132020831,
|
||
|
|
"step": 210
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.245067394648989e-05,
|
||
|
|
"clip_ratio/high_mean": 3.245067394648989e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00036193533580646745,
|
||
|
|
"clip_ratio/low_min": 0.00036193533580646745,
|
||
|
|
"clip_ratio/region_mean": 0.0003943860097529574,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 72.0,
|
||
|
|
"completions/max_terminated_length": 72.0,
|
||
|
|
"completions/mean_length": 53.7109375,
|
||
|
|
"completions/mean_terminated_length": 53.7109375,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.12079384653932518,
|
||
|
|
"epoch": 0.05654338549075391,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.6931771039962769,
|
||
|
|
"kl": 0.210977077898052,
|
||
|
|
"learning_rate": 2.8768461914473794e-07,
|
||
|
|
"loss": 0.0021,
|
||
|
|
"num_tokens": 14281181.0,
|
||
|
|
"reward": 1.205078125,
|
||
|
|
"reward_std": 0.143476203083992,
|
||
|
|
"rewards/equation_reward_func/mean": 0.205078125,
|
||
|
|
"rewards/equation_reward_func/std": 0.4041535556316376,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8660997152328491,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000596046447754,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6091339588165283,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6238505840301514,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005528800189495087,
|
||
|
|
"step": 212
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012854254843356708,
|
||
|
|
"clip_ratio/high_mean": 0.00012854254843356708,
|
||
|
|
"clip_ratio/low_mean": 0.00019423514863269197,
|
||
|
|
"clip_ratio/low_min": 0.00019423514863269197,
|
||
|
|
"clip_ratio/region_mean": 0.00032277769706625905,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 71.0,
|
||
|
|
"completions/max_terminated_length": 71.0,
|
||
|
|
"completions/mean_length": 53.357421875,
|
||
|
|
"completions/mean_terminated_length": 53.357421875,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11314595283733474,
|
||
|
|
"epoch": 0.057076813655761026,
|
||
|
|
"frac_reward_zero_std": 0.609375,
|
||
|
|
"grad_norm": 0.5694147348403931,
|
||
|
|
"kl": 0.28766577939192456,
|
||
|
|
"learning_rate": 2.8411925163961926e-07,
|
||
|
|
"loss": -0.0096,
|
||
|
|
"num_tokens": 14386692.0,
|
||
|
|
"reward": 1.146484375,
|
||
|
|
"reward_std": 0.1631450653076172,
|
||
|
|
"rewards/equation_reward_func/mean": 0.150390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.35780346393585205,
|
||
|
|
"rewards/format_reward_func/mean": 0.99609375,
|
||
|
|
"rewards/format_reward_func/std": 0.06243881583213806,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6400882005691528,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001121759414673,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6106104254722595,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4947500228881836,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005793167278170586,
|
||
|
|
"step": 214
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.60305559500638e-05,
|
||
|
|
"clip_ratio/high_mean": 9.60305559500638e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0002618474051511536,
|
||
|
|
"clip_ratio/low_min": 0.0002618474051511536,
|
||
|
|
"clip_ratio/region_mean": 0.0003578779611012174,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 72.0,
|
||
|
|
"completions/max_terminated_length": 72.0,
|
||
|
|
"completions/mean_length": 52.5625,
|
||
|
|
"completions/mean_terminated_length": 52.5625,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.11545825170146094,
|
||
|
|
"epoch": 0.057610241820768134,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.8972283005714417,
|
||
|
|
"kl": 0.4845621803154548,
|
||
|
|
"learning_rate": 2.8054679850011825e-07,
|
||
|
|
"loss": 0.0024,
|
||
|
|
"num_tokens": 14491556.0,
|
||
|
|
"reward": 1.1328125,
|
||
|
|
"reward_std": 0.1345345377922058,
|
||
|
|
"rewards/equation_reward_func/mean": 0.13671875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3438861668109894,
|
||
|
|
"rewards/format_reward_func/mean": 0.99609375,
|
||
|
|
"rewards/format_reward_func/std": 0.06243881583213806,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.590267539024353,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001193284988403,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5377895832061768,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6202878952026367,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0057983691804111,
|
||
|
|
"step": 216
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001974168214170883,
|
||
|
|
"clip_ratio/high_mean": 0.0001974168214170883,
|
||
|
|
"clip_ratio/low_mean": 6.495545838131673e-05,
|
||
|
|
"clip_ratio/low_min": 6.495545838131673e-05,
|
||
|
|
"clip_ratio/region_mean": 0.000262372279798405,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 77.0,
|
||
|
|
"completions/max_terminated_length": 77.0,
|
||
|
|
"completions/mean_length": 52.99609375,
|
||
|
|
"completions/mean_terminated_length": 52.99609375,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11492264664007558,
|
||
|
|
"epoch": 0.05814366998577525,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.4676933288574219,
|
||
|
|
"kl": 0.21251611121826702,
|
||
|
|
"learning_rate": 2.769680016269385e-07,
|
||
|
|
"loss": -0.0036,
|
||
|
|
"num_tokens": 14596754.0,
|
||
|
|
"reward": 1.1171875,
|
||
|
|
"reward_std": 0.11875930428504944,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1171875,
|
||
|
|
"rewards/equation_reward_func/std": 0.32195815443992615,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5096341371536255,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998860955238342,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5001150369644165,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6929171085357666,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005857203155755997,
|
||
|
|
"step": 218
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00013001335901208222,
|
||
|
|
"clip_ratio/high_mean": 0.00013001335901208222,
|
||
|
|
"clip_ratio/low_mean": 9.567012117865185e-05,
|
||
|
|
"clip_ratio/low_min": 9.567012117865185e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00022568348019073406,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 73.0,
|
||
|
|
"completions/max_terminated_length": 73.0,
|
||
|
|
"completions/mean_length": 53.59765625,
|
||
|
|
"completions/mean_terminated_length": 53.59765625,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.11644557221896118,
|
||
|
|
"epoch": 0.05867709815078236,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.5575525164604187,
|
||
|
|
"kl": 0.19831780116591188,
|
||
|
|
"learning_rate": 2.7338360423820327e-07,
|
||
|
|
"loss": 0.0063,
|
||
|
|
"num_tokens": 14702532.0,
|
||
|
|
"reward": 1.09375,
|
||
|
|
"reward_std": 0.12684723734855652,
|
||
|
|
"rewards/equation_reward_func/mean": 0.09765625,
|
||
|
|
"rewards/equation_reward_func/std": 0.29713961482048035,
|
||
|
|
"rewards/format_reward_func/mean": 0.99609375,
|
||
|
|
"rewards/format_reward_func/std": 0.06243881583213806,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5924863815307617,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9996614456176758,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47730016708374023,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7396097183227539,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005882314871996641,
|
||
|
|
"step": 220
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0003587727583686097,
|
||
|
|
"clip_ratio/high_mean": 0.0003587727583686097,
|
||
|
|
"clip_ratio/low_mean": 0.00016187850592864884,
|
||
|
|
"clip_ratio/low_min": 0.00016187850592864884,
|
||
|
|
"clip_ratio/region_mean": 0.0005206512642972586,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 53.50390625,
|
||
|
|
"completions/mean_terminated_length": 53.50390625,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.11813913865221871,
|
||
|
|
"epoch": 0.05921052631578947,
|
||
|
|
"frac_reward_zero_std": 0.625,
|
||
|
|
"grad_norm": 1.0590949058532715,
|
||
|
|
"kl": 0.3677462774018447,
|
||
|
|
"learning_rate": 2.6979435071510956e-07,
|
||
|
|
"loss": -0.0064,
|
||
|
|
"num_tokens": 14808158.0,
|
||
|
|
"reward": 1.125,
|
||
|
|
"reward_std": 0.15932825207710266,
|
||
|
|
"rewards/equation_reward_func/mean": 0.125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3310423493385315,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5182225704193115,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999364614486694,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5411150455474854,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6141233444213867,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006206444930285215,
|
||
|
|
"step": 222
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0003639115958422836,
|
||
|
|
"clip_ratio/high_mean": 0.0003639115958422836,
|
||
|
|
"clip_ratio/low_mean": 0.00020010410096599825,
|
||
|
|
"clip_ratio/low_min": 0.00020010410096599825,
|
||
|
|
"clip_ratio/region_mean": 0.0005640156968082818,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 82.0,
|
||
|
|
"completions/max_terminated_length": 82.0,
|
||
|
|
"completions/mean_length": 52.18359375,
|
||
|
|
"completions/mean_terminated_length": 52.18359375,
|
||
|
|
"completions/min_length": 33.0,
|
||
|
|
"completions/min_terminated_length": 33.0,
|
||
|
|
"entropy": 0.11953117636342843,
|
||
|
|
"epoch": 0.059743954480796585,
|
||
|
|
"frac_reward_zero_std": 0.59375,
|
||
|
|
"grad_norm": 0.705763578414917,
|
||
|
|
"kl": 0.19684387784865168,
|
||
|
|
"learning_rate": 2.662009864473406e-07,
|
||
|
|
"loss": 0.0075,
|
||
|
|
"num_tokens": 14912820.0,
|
||
|
|
"reward": 1.150390625,
|
||
|
|
"reward_std": 0.18030321598052979,
|
||
|
|
"rewards/equation_reward_func/mean": 0.150390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.35780346393585205,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6097370386123657,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001099109649658,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4614678919315338,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7733428478240967,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006311375647783279,
|
||
|
|
"step": 224
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.85095975920558e-05,
|
||
|
|
"clip_ratio/high_mean": 9.85095975920558e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00022559486226075224,
|
||
|
|
"clip_ratio/low_min": 0.00022559486226075224,
|
||
|
|
"clip_ratio/region_mean": 0.00032410445985280804,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 75.0,
|
||
|
|
"completions/max_terminated_length": 75.0,
|
||
|
|
"completions/mean_length": 52.91015625,
|
||
|
|
"completions/mean_terminated_length": 52.91015625,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.11986736642817657,
|
||
|
|
"epoch": 0.0602773826458037,
|
||
|
|
"frac_reward_zero_std": 0.6640625,
|
||
|
|
"grad_norm": 0.6192528605461121,
|
||
|
|
"kl": 0.24484028832779992,
|
||
|
|
"learning_rate": 2.626042576782687e-07,
|
||
|
|
"loss": 0.0061,
|
||
|
|
"num_tokens": 15123128.0,
|
||
|
|
"reward": 1.134765625,
|
||
|
|
"reward_std": 0.14946477115154266,
|
||
|
|
"rewards/equation_reward_func/mean": 0.134765625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3403467684984207,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7585115432739258,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001845359802246,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5394621193408966,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6830110549926758,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.00615977100096643,
|
||
|
|
"step": 226
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0002973029719820867,
|
||
|
|
"clip_ratio/high_mean": 0.0002973029719820867,
|
||
|
|
"clip_ratio/low_mean": 6.856257742684748e-05,
|
||
|
|
"clip_ratio/low_min": 6.856257742684748e-05,
|
||
|
|
"clip_ratio/region_mean": 0.0003658655494089342,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 71.0,
|
||
|
|
"completions/max_terminated_length": 71.0,
|
||
|
|
"completions/mean_length": 53.609375,
|
||
|
|
"completions/mean_terminated_length": 53.609375,
|
||
|
|
"completions/min_length": 31.0,
|
||
|
|
"completions/min_terminated_length": 31.0,
|
||
|
|
"entropy": 0.11842664269109567,
|
||
|
|
"epoch": 0.060810810810810814,
|
||
|
|
"frac_reward_zero_std": 0.734375,
|
||
|
|
"grad_norm": 0.708135187625885,
|
||
|
|
"kl": 0.1876696693814463,
|
||
|
|
"learning_rate": 2.590049113499809e-07,
|
||
|
|
"loss": -0.0091,
|
||
|
|
"num_tokens": 15228832.0,
|
||
|
|
"reward": 1.091796875,
|
||
|
|
"reward_std": 0.11895088851451874,
|
||
|
|
"rewards/equation_reward_func/mean": 0.091796875,
|
||
|
|
"rewards/equation_reward_func/std": 0.289021372795105,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6424556970596313,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999076128005981,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47103625535964966,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7528202533721924,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006564016919583082,
|
||
|
|
"step": 228
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.000291443365212116,
|
||
|
|
"clip_ratio/high_mean": 0.000291443365212116,
|
||
|
|
"clip_ratio/low_mean": 6.583160332714517e-05,
|
||
|
|
"clip_ratio/low_min": 6.583160332714517e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00035727496853926114,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 69.0,
|
||
|
|
"completions/max_terminated_length": 69.0,
|
||
|
|
"completions/mean_length": 52.140625,
|
||
|
|
"completions/mean_terminated_length": 52.140625,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.12662849927114117,
|
||
|
|
"epoch": 0.06134423897581792,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.47451645135879517,
|
||
|
|
"kl": 0.2256828004287349,
|
||
|
|
"learning_rate": 2.5540369494815966e-07,
|
||
|
|
"loss": -0.0013,
|
||
|
|
"num_tokens": 15333496.0,
|
||
|
|
"reward": 1.126953125,
|
||
|
|
"reward_std": 0.1261277049779892,
|
||
|
|
"rewards/equation_reward_func/mean": 0.12890625,
|
||
|
|
"rewards/equation_reward_func/std": 0.33542385697364807,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6544182300567627,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0002301931381226,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5614175796508789,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5772902965545654,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.00661100260913372,
|
||
|
|
"step": 230
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00010160589590668678,
|
||
|
|
"clip_ratio/high_mean": 0.00010160589590668678,
|
||
|
|
"clip_ratio/low_mean": 6.522905379016366e-05,
|
||
|
|
"clip_ratio/low_min": 6.522905379016366e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00016683494969685044,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 71.0,
|
||
|
|
"completions/max_terminated_length": 71.0,
|
||
|
|
"completions/mean_length": 52.98828125,
|
||
|
|
"completions/mean_terminated_length": 52.98828125,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.12193829255799453,
|
||
|
|
"epoch": 0.061877667140825036,
|
||
|
|
"frac_reward_zero_std": 0.734375,
|
||
|
|
"grad_norm": 0.5908534526824951,
|
||
|
|
"kl": 0.22970337400005925,
|
||
|
|
"learning_rate": 2.5180135634685064e-07,
|
||
|
|
"loss": 0.0012,
|
||
|
|
"num_tokens": 15438778.0,
|
||
|
|
"reward": 1.1171875,
|
||
|
|
"reward_std": 0.11980462074279785,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1171875,
|
||
|
|
"rewards/equation_reward_func/std": 0.32195815443992615,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5204116106033325,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997380971908569,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.49725833535194397,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6986455917358398,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006525573320686817,
|
||
|
|
"step": 232
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00016493863788329891,
|
||
|
|
"clip_ratio/high_mean": 0.00016493863788329891,
|
||
|
|
"clip_ratio/low_mean": 0.0001652783765974972,
|
||
|
|
"clip_ratio/low_min": 0.0001652783765974972,
|
||
|
|
"clip_ratio/region_mean": 0.0003302170177145551,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 52.349609375,
|
||
|
|
"completions/mean_terminated_length": 52.349609375,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.1254527090738217,
|
||
|
|
"epoch": 0.06241109530583215,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.6845598816871643,
|
||
|
|
"kl": 0.1800268271731006,
|
||
|
|
"learning_rate": 2.4819864365314934e-07,
|
||
|
|
"loss": 0.0079,
|
||
|
|
"num_tokens": 15543501.0,
|
||
|
|
"reward": 1.16796875,
|
||
|
|
"reward_std": 0.13776159286499023,
|
||
|
|
"rewards/equation_reward_func/mean": 0.16796875,
|
||
|
|
"rewards/equation_reward_func/std": 0.374204158782959,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5828297138214111,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000268578529358,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5083870887756348,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6765121221542358,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.00637659290805459,
|
||
|
|
"step": 234
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00019972173807521662,
|
||
|
|
"clip_ratio/high_mean": 0.00019972173807521662,
|
||
|
|
"clip_ratio/low_mean": 6.456824808588458e-05,
|
||
|
|
"clip_ratio/low_min": 6.456824808588458e-05,
|
||
|
|
"clip_ratio/region_mean": 0.0002642899861611012,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 71.0,
|
||
|
|
"completions/max_terminated_length": 71.0,
|
||
|
|
"completions/mean_length": 53.46484375,
|
||
|
|
"completions/mean_terminated_length": 53.46484375,
|
||
|
|
"completions/min_length": 29.0,
|
||
|
|
"completions/min_terminated_length": 29.0,
|
||
|
|
"entropy": 0.12084446619782183,
|
||
|
|
"epoch": 0.06294452347083926,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 0.6229382753372192,
|
||
|
|
"kl": 0.32917360506123966,
|
||
|
|
"learning_rate": 2.445963050518403e-07,
|
||
|
|
"loss": -0.0,
|
||
|
|
"num_tokens": 15649187.0,
|
||
|
|
"reward": 1.13671875,
|
||
|
|
"reward_std": 0.1554545760154724,
|
||
|
|
"rewards/equation_reward_func/mean": 0.138671875,
|
||
|
|
"rewards/equation_reward_func/std": 0.34594178199768066,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5042505264282227,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.999774694442749,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4756012558937073,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7431755065917969,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006509827449917793,
|
||
|
|
"step": 236
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.2075955661841566e-05,
|
||
|
|
"clip_ratio/high_mean": 3.2075955661841566e-05,
|
||
|
|
"clip_ratio/low_mean": 3.1855248380452394e-05,
|
||
|
|
"clip_ratio/low_min": 3.1855248380452394e-05,
|
||
|
|
"clip_ratio/region_mean": 6.393120404229396e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 75.0,
|
||
|
|
"completions/max_terminated_length": 75.0,
|
||
|
|
"completions/mean_length": 52.603515625,
|
||
|
|
"completions/mean_terminated_length": 52.603515625,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.12392547105749448,
|
||
|
|
"epoch": 0.06347795163584637,
|
||
|
|
"frac_reward_zero_std": 0.734375,
|
||
|
|
"grad_norm": 0.5728958249092102,
|
||
|
|
"kl": 0.22863260573811,
|
||
|
|
"learning_rate": 2.4099508865001914e-07,
|
||
|
|
"loss": -0.004,
|
||
|
|
"num_tokens": 15754232.0,
|
||
|
|
"reward": 1.15234375,
|
||
|
|
"reward_std": 0.12145397812128067,
|
||
|
|
"rewards/equation_reward_func/mean": 0.15234375,
|
||
|
|
"rewards/equation_reward_func/std": 0.35970520973205566,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4579753875732422,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001283884048462,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47823217511177063,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7376589775085449,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006175911519676447,
|
||
|
|
"step": 238
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.573096758686006e-05,
|
||
|
|
"clip_ratio/high_mean": 9.573096758686006e-05,
|
||
|
|
"clip_ratio/low_mean": 9.874854650762345e-05,
|
||
|
|
"clip_ratio/low_min": 9.874854650762345e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00019447951409448352,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 68.0,
|
||
|
|
"completions/max_terminated_length": 68.0,
|
||
|
|
"completions/mean_length": 53.09375,
|
||
|
|
"completions/mean_terminated_length": 53.09375,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.117175265111857,
|
||
|
|
"epoch": 0.06401137980085349,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.5250370502471924,
|
||
|
|
"kl": 0.28859515984853107,
|
||
|
|
"learning_rate": 2.3739574232173134e-07,
|
||
|
|
"loss": 0.0005,
|
||
|
|
"num_tokens": 15859560.0,
|
||
|
|
"reward": 1.162109375,
|
||
|
|
"reward_std": 0.14729177951812744,
|
||
|
|
"rewards/equation_reward_func/mean": 0.162109375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3689115643501282,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5415260791778564,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000598430633545,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5174270868301392,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6588866710662842,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006172684486955404,
|
||
|
|
"step": 240
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.264133637357089e-05,
|
||
|
|
"clip_ratio/high_mean": 3.264133637357089e-05,
|
||
|
|
"clip_ratio/low_mean": 9.471062609615426e-05,
|
||
|
|
"clip_ratio/low_min": 9.471062609615426e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00012735196246972514,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 71.5,
|
||
|
|
"completions/max_terminated_length": 71.5,
|
||
|
|
"completions/mean_length": 53.861328125,
|
||
|
|
"completions/mean_terminated_length": 53.861328125,
|
||
|
|
"completions/min_length": 36.5,
|
||
|
|
"completions/min_terminated_length": 36.5,
|
||
|
|
"entropy": 0.12318918564253384,
|
||
|
|
"epoch": 0.0645448079658606,
|
||
|
|
"frac_reward_zero_std": 0.734375,
|
||
|
|
"grad_norm": 0.4942455589771271,
|
||
|
|
"kl": 0.23004327176345718,
|
||
|
|
"learning_rate": 2.3379901355265936e-07,
|
||
|
|
"loss": 0.0022,
|
||
|
|
"num_tokens": 16071202.0,
|
||
|
|
"reward": 1.1396484375,
|
||
|
|
"reward_std": 0.11553288251161575,
|
||
|
|
"rewards/equation_reward_func/mean": 0.140625,
|
||
|
|
"rewards/equation_reward_func/std": 0.34502755105495453,
|
||
|
|
"rewards/format_reward_func/mean": 0.9990234375,
|
||
|
|
"rewards/format_reward_func/std": 0.022097086533904076,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4460033774375916,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000775754451752,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5052160322666168,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7033942937850952,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006043520988896489,
|
||
|
|
"step": 242
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.313164097567399e-05,
|
||
|
|
"clip_ratio/high_mean": 6.313164097567399e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00012851631294728981,
|
||
|
|
"clip_ratio/low_min": 0.00012851631294728981,
|
||
|
|
"clip_ratio/region_mean": 0.00019164795392296382,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 81.0,
|
||
|
|
"completions/max_terminated_length": 81.0,
|
||
|
|
"completions/mean_length": 53.490234375,
|
||
|
|
"completions/mean_terminated_length": 53.490234375,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.12224643304944038,
|
||
|
|
"epoch": 0.0650782361308677,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.5327535271644592,
|
||
|
|
"kl": 0.1954546593543556,
|
||
|
|
"learning_rate": 2.3020564928489041e-07,
|
||
|
|
"loss": -0.0005,
|
||
|
|
"num_tokens": 16176717.0,
|
||
|
|
"reward": 1.09375,
|
||
|
|
"reward_std": 0.12671303749084473,
|
||
|
|
"rewards/equation_reward_func/mean": 0.09375,
|
||
|
|
"rewards/equation_reward_func/std": 0.29176566004753113,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5413131713867188,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001628398895264,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6364158391952515,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4519031047821045,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006325087510049343,
|
||
|
|
"step": 244
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.61559428812729e-05,
|
||
|
|
"clip_ratio/high_mean": 6.61559428812729e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00016481440333235596,
|
||
|
|
"clip_ratio/low_min": 0.00016481440333235596,
|
||
|
|
"clip_ratio/region_mean": 0.00023097034621362886,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 52.734375,
|
||
|
|
"completions/mean_terminated_length": 52.734375,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.12224224023520947,
|
||
|
|
"epoch": 0.06561166429587482,
|
||
|
|
"frac_reward_zero_std": 0.734375,
|
||
|
|
"grad_norm": 0.5621053576469421,
|
||
|
|
"kl": 0.2635859563532803,
|
||
|
|
"learning_rate": 2.2661639576179676e-07,
|
||
|
|
"loss": -0.0083,
|
||
|
|
"num_tokens": 16281669.0,
|
||
|
|
"reward": 1.173828125,
|
||
|
|
"reward_std": 0.11270461976528168,
|
||
|
|
"rewards/equation_reward_func/mean": 0.17578125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3810062110424042,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4159756898880005,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000406503677368,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5447098612785339,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6075019836425781,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005927799269556999,
|
||
|
|
"step": 246
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00013626467100241117,
|
||
|
|
"clip_ratio/high_mean": 0.00013626467100241117,
|
||
|
|
"clip_ratio/low_mean": 0.0002326449547480378,
|
||
|
|
"clip_ratio/low_min": 0.0002326449547480378,
|
||
|
|
"clip_ratio/region_mean": 0.00036890962575044896,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 78.0,
|
||
|
|
"completions/max_terminated_length": 78.0,
|
||
|
|
"completions/mean_length": 52.96484375,
|
||
|
|
"completions/mean_terminated_length": 52.96484375,
|
||
|
|
"completions/min_length": 30.0,
|
||
|
|
"completions/min_terminated_length": 30.0,
|
||
|
|
"entropy": 0.12200754197935264,
|
||
|
|
"epoch": 0.06614509246088193,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 1.1268446445465088,
|
||
|
|
"kl": 0.28940939489338136,
|
||
|
|
"learning_rate": 2.2303199837306153e-07,
|
||
|
|
"loss": 0.0127,
|
||
|
|
"num_tokens": 16386787.0,
|
||
|
|
"reward": 1.138671875,
|
||
|
|
"reward_std": 0.1637316346168518,
|
||
|
|
"rewards/equation_reward_func/mean": 0.140625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3479743003845215,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6521880626678467,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000035047531128,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5477961897850037,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6018519401550293,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006177326198667288,
|
||
|
|
"step": 248
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.517285105979277e-05,
|
||
|
|
"clip_ratio/high_mean": 6.517285105979277e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00036028153004331724,
|
||
|
|
"clip_ratio/low_min": 0.00036028153004331724,
|
||
|
|
"clip_ratio/region_mean": 0.00042545438110311,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 70.0,
|
||
|
|
"completions/max_terminated_length": 70.0,
|
||
|
|
"completions/mean_length": 52.390625,
|
||
|
|
"completions/mean_terminated_length": 52.390625,
|
||
|
|
"completions/min_length": 33.0,
|
||
|
|
"completions/min_terminated_length": 33.0,
|
||
|
|
"entropy": 0.1172566707763407,
|
||
|
|
"epoch": 0.06667852062588904,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.5600212216377258,
|
||
|
|
"kl": 0.22640964802768496,
|
||
|
|
"learning_rate": 2.194532014998817e-07,
|
||
|
|
"loss": -0.0154,
|
||
|
|
"num_tokens": 16491555.0,
|
||
|
|
"reward": 1.0859375,
|
||
|
|
"reward_std": 0.1346687376499176,
|
||
|
|
"rewards/equation_reward_func/mean": 0.087890625,
|
||
|
|
"rewards/equation_reward_func/std": 0.2834126651287079,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.541048288345337,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998010396957397,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4184038043022156,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8713083267211914,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005250845570117235,
|
||
|
|
"step": 250
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00013368520174278982,
|
||
|
|
"clip_ratio/high_mean": 0.00013368520174278982,
|
||
|
|
"clip_ratio/low_mean": 0.00043396216480889253,
|
||
|
|
"clip_ratio/low_min": 0.00043396216480889253,
|
||
|
|
"clip_ratio/region_mean": 0.0005676473665516824,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 69.0,
|
||
|
|
"completions/max_terminated_length": 69.0,
|
||
|
|
"completions/mean_length": 53.234375,
|
||
|
|
"completions/mean_terminated_length": 53.234375,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11685328806440036,
|
||
|
|
"epoch": 0.06721194879089616,
|
||
|
|
"frac_reward_zero_std": 0.625,
|
||
|
|
"grad_norm": 0.6178267002105713,
|
||
|
|
"kl": 0.48353597418301636,
|
||
|
|
"learning_rate": 2.1588074836038071e-07,
|
||
|
|
"loss": 0.0135,
|
||
|
|
"num_tokens": 16596835.0,
|
||
|
|
"reward": 1.1484375,
|
||
|
|
"reward_std": 0.16728200018405914,
|
||
|
|
"rewards/equation_reward_func/mean": 0.150390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.35780346393585205,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5711119174957275,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999057650566101,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6018173098564148,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5078012943267822,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005669458769261837,
|
||
|
|
"step": 252
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00013041867189005844,
|
||
|
|
"clip_ratio/high_mean": 0.00013041867189005844,
|
||
|
|
"clip_ratio/low_mean": 0.0001985825365409255,
|
||
|
|
"clip_ratio/low_min": 0.0001985825365409255,
|
||
|
|
"clip_ratio/region_mean": 0.00032900120843098394,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 81.0,
|
||
|
|
"completions/max_terminated_length": 81.0,
|
||
|
|
"completions/mean_length": 53.201171875,
|
||
|
|
"completions/mean_terminated_length": 53.201171875,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11372616328299046,
|
||
|
|
"epoch": 0.06774537695590327,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.6195980310440063,
|
||
|
|
"kl": 0.20957040414214134,
|
||
|
|
"learning_rate": 2.1231538085526204e-07,
|
||
|
|
"loss": -0.0052,
|
||
|
|
"num_tokens": 16702090.0,
|
||
|
|
"reward": 1.173828125,
|
||
|
|
"reward_std": 0.14814674854278564,
|
||
|
|
"rewards/equation_reward_func/mean": 0.173828125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3793322443962097,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4368383884429932,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000479221343994,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5573049187660217,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5846426486968994,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005298153031617403,
|
||
|
|
"step": 254
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00016930111774450375,
|
||
|
|
"clip_ratio/high_mean": 0.00016930111774450375,
|
||
|
|
"clip_ratio/low_mean": 0.0002204755631585916,
|
||
|
|
"clip_ratio/low_min": 0.0002204755631585916,
|
||
|
|
"clip_ratio/region_mean": 0.0003897766776693364,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 87.0,
|
||
|
|
"completions/max_terminated_length": 87.0,
|
||
|
|
"completions/mean_length": 54.55078125,
|
||
|
|
"completions/mean_terminated_length": 54.55078125,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.12089426215324137,
|
||
|
|
"epoch": 0.06827880512091039,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.7291125059127808,
|
||
|
|
"kl": 0.2326575161682235,
|
||
|
|
"learning_rate": 2.0875783941373686e-07,
|
||
|
|
"loss": 0.0027,
|
||
|
|
"num_tokens": 16808332.0,
|
||
|
|
"reward": 1.115234375,
|
||
|
|
"reward_std": 0.1349339485168457,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1171875,
|
||
|
|
"rewards/equation_reward_func/std": 0.32195815443992615,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.592735767364502,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001572370529175,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6112369894981384,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.49227046966552734,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006166752427816391,
|
||
|
|
"step": 256
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.400825805030763e-05,
|
||
|
|
"clip_ratio/high_mean": 6.400825805030763e-05,
|
||
|
|
"clip_ratio/low_mean": 6.572409701119695e-05,
|
||
|
|
"clip_ratio/low_min": 6.572409701119695e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00012973235506150458,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 77.0,
|
||
|
|
"completions/max_terminated_length": 77.0,
|
||
|
|
"completions/mean_length": 53.673828125,
|
||
|
|
"completions/mean_terminated_length": 53.673828125,
|
||
|
|
"completions/min_length": 35.5,
|
||
|
|
"completions/min_terminated_length": 35.5,
|
||
|
|
"entropy": 0.11469285790291098,
|
||
|
|
"epoch": 0.0688122332859175,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.6034818291664124,
|
||
|
|
"kl": 0.21051878440711233,
|
||
|
|
"learning_rate": 2.052088628397572e-07,
|
||
|
|
"loss": 0.0046,
|
||
|
|
"num_tokens": 17019478.0,
|
||
|
|
"reward": 1.140625,
|
||
|
|
"reward_std": 0.14814674109220505,
|
||
|
|
"rewards/equation_reward_func/mean": 0.140625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3478719890117645,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6311516761779785,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000055432319641,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5794435143470764,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5485678911209106,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005461497465148568,
|
||
|
|
"step": 258
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00019554619858455326,
|
||
|
|
"clip_ratio/high_mean": 0.00019554619858455326,
|
||
|
|
"clip_ratio/low_mean": 9.79065597574744e-05,
|
||
|
|
"clip_ratio/low_min": 9.79065597574744e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00029345275834202766,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 71.0,
|
||
|
|
"completions/max_terminated_length": 71.0,
|
||
|
|
"completions/mean_length": 53.521484375,
|
||
|
|
"completions/mean_terminated_length": 53.521484375,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11641324021750027,
|
||
|
|
"epoch": 0.0693456614509246,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.5642570853233337,
|
||
|
|
"kl": 0.2255418917371167,
|
||
|
|
"learning_rate": 2.0166918815858688e-07,
|
||
|
|
"loss": -0.0029,
|
||
|
|
"num_tokens": 17124913.0,
|
||
|
|
"reward": 1.16796875,
|
||
|
|
"reward_std": 0.14190170168876648,
|
||
|
|
"rewards/equation_reward_func/mean": 0.16796875,
|
||
|
|
"rewards/equation_reward_func/std": 0.374204158782959,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7392843961715698,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999298453330994,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4831935167312622,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7273380756378174,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005574048962444067,
|
||
|
|
"step": 260
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 0.0002266580381223725,
|
||
|
|
"clip_ratio/low_min": 0.0002266580381223725,
|
||
|
|
"clip_ratio/region_mean": 0.0002266580381223725,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 70.0,
|
||
|
|
"completions/max_terminated_length": 70.0,
|
||
|
|
"completions/mean_length": 54.185546875,
|
||
|
|
"completions/mean_terminated_length": 54.185546875,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11455756094720629,
|
||
|
|
"epoch": 0.06987908961593173,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 0.550462543964386,
|
||
|
|
"kl": 0.17767776879999372,
|
||
|
|
"learning_rate": 1.9813955046374102e-07,
|
||
|
|
"loss": -0.0048,
|
||
|
|
"num_tokens": 17230824.0,
|
||
|
|
"reward": 1.1953125,
|
||
|
|
"reward_std": 0.1442737877368927,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1953125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3968288004398346,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8483010530471802,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.999902606010437,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5373868346214294,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6210370063781738,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005659033544361591,
|
||
|
|
"step": 262
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.5365269417978e-05,
|
||
|
|
"clip_ratio/high_mean": 6.5365269417978e-05,
|
||
|
|
"clip_ratio/low_mean": 9.701474078206552e-05,
|
||
|
|
"clip_ratio/low_min": 9.701474078206552e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00016238001020004353,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 77.0,
|
||
|
|
"completions/max_terminated_length": 77.0,
|
||
|
|
"completions/mean_length": 53.888671875,
|
||
|
|
"completions/mean_terminated_length": 53.888671875,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11500472492641872,
|
||
|
|
"epoch": 0.07041251778093884,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.7422778010368347,
|
||
|
|
"kl": 0.2729740840279394,
|
||
|
|
"learning_rate": 1.946206827643275e-07,
|
||
|
|
"loss": 0.0041,
|
||
|
|
"num_tokens": 17336551.0,
|
||
|
|
"reward": 1.181640625,
|
||
|
|
"reward_std": 0.1434762179851532,
|
||
|
|
"rewards/equation_reward_func/mean": 0.181640625,
|
||
|
|
"rewards/equation_reward_func/std": 0.38592514395713806,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5066125392913818,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999542832374573,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.42032745480537415,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8667211532592773,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005726358853280544,
|
||
|
|
"step": 264
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012987014876368144,
|
||
|
|
"clip_ratio/high_mean": 0.00012987014876368144,
|
||
|
|
"clip_ratio/low_mean": 0.00012680440947103003,
|
||
|
|
"clip_ratio/low_min": 0.00012680440947103003,
|
||
|
|
"clip_ratio/region_mean": 0.00025667455823471147,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 72.0,
|
||
|
|
"completions/max_terminated_length": 72.0,
|
||
|
|
"completions/mean_length": 54.5390625,
|
||
|
|
"completions/mean_terminated_length": 54.5390625,
|
||
|
|
"completions/min_length": 25.0,
|
||
|
|
"completions/min_terminated_length": 25.0,
|
||
|
|
"entropy": 0.11781905467311542,
|
||
|
|
"epoch": 0.07094594594594594,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.5870804190635681,
|
||
|
|
"kl": 0.18947985106044346,
|
||
|
|
"learning_rate": 1.9111331583282103e-07,
|
||
|
|
"loss": 0.0051,
|
||
|
|
"num_tokens": 17442779.0,
|
||
|
|
"reward": 1.12890625,
|
||
|
|
"reward_std": 0.13250252604484558,
|
||
|
|
"rewards/equation_reward_func/mean": 0.12890625,
|
||
|
|
"rewards/equation_reward_func/std": 0.33542385697364807,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.785035490989685,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999924302101135,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47538992762565613,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7436199188232422,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005903167650103569,
|
||
|
|
"step": 266
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012978605809621513,
|
||
|
|
"clip_ratio/high_mean": 0.00012978605809621513,
|
||
|
|
"clip_ratio/low_mean": 0.00015856400972956585,
|
||
|
|
"clip_ratio/low_min": 0.00015856400972956585,
|
||
|
|
"clip_ratio/region_mean": 0.00028835006782578095,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 54.20703125,
|
||
|
|
"completions/mean_terminated_length": 54.20703125,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.11669882759451866,
|
||
|
|
"epoch": 0.07147937411095306,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.7162891626358032,
|
||
|
|
"kl": 0.2612205685840713,
|
||
|
|
"learning_rate": 1.8761817805330195e-07,
|
||
|
|
"loss": -0.0032,
|
||
|
|
"num_tokens": 17548757.0,
|
||
|
|
"reward": 1.115234375,
|
||
|
|
"reward_std": 0.12164750695228577,
|
||
|
|
"rewards/equation_reward_func/mean": 0.115234375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3196168541908264,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7249046564102173,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998591542243958,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4793323278427124,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7353610992431641,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006094635929912329,
|
||
|
|
"step": 268
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0005107695793008639,
|
||
|
|
"clip_ratio/high_mean": 0.0005107695793008639,
|
||
|
|
"clip_ratio/low_mean": 0.00022279931646254327,
|
||
|
|
"clip_ratio/low_min": 0.00022279931646254327,
|
||
|
|
"clip_ratio/region_mean": 0.000733568902230925,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 75.0,
|
||
|
|
"completions/max_terminated_length": 75.0,
|
||
|
|
"completions/mean_length": 54.283203125,
|
||
|
|
"completions/mean_terminated_length": 54.283203125,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11902402403453986,
|
||
|
|
"epoch": 0.07201280227596017,
|
||
|
|
"frac_reward_zero_std": 0.734375,
|
||
|
|
"grad_norm": 0.37476852536201477,
|
||
|
|
"kl": 0.22781566406289736,
|
||
|
|
"learning_rate": 1.8413599527019018e-07,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 17654854.0,
|
||
|
|
"reward": 1.146484375,
|
||
|
|
"reward_std": 0.12348721921443939,
|
||
|
|
"rewards/equation_reward_func/mean": 0.146484375,
|
||
|
|
"rewards/equation_reward_func/std": 0.35393697023391724,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.42414128780365,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999381899833679,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.3701840937137604,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.9937548637390137,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0060479482635855675,
|
||
|
|
"step": 270
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.0358228185731503e-05,
|
||
|
|
"clip_ratio/high_mean": 3.0358228185731503e-05,
|
||
|
|
"clip_ratio/low_mean": 6.29265883213116e-05,
|
||
|
|
"clip_ratio/low_min": 6.29265883213116e-05,
|
||
|
|
"clip_ratio/region_mean": 9.328481650704311e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 55.328125,
|
||
|
|
"completions/mean_terminated_length": 55.328125,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.1182148756666316,
|
||
|
|
"epoch": 0.07254623044096728,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 0.7100948095321655,
|
||
|
|
"kl": 0.2611344123466147,
|
||
|
|
"learning_rate": 1.806674906375079e-07,
|
||
|
|
"loss": -0.0018,
|
||
|
|
"num_tokens": 17761686.0,
|
||
|
|
"reward": 1.140625,
|
||
|
|
"reward_std": 0.15012383460998535,
|
||
|
|
"rewards/equation_reward_func/mean": 0.142578125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3499840497970581,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5610944032669067,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999486804008484,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5355713367462158,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6244211196899414,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005923059768974781,
|
||
|
|
"step": 272
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 6.709608068275783e-05,
|
||
|
|
"clip_ratio/low_min": 6.709608068275783e-05,
|
||
|
|
"clip_ratio/region_mean": 6.709608068275783e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 75.5,
|
||
|
|
"completions/max_terminated_length": 75.5,
|
||
|
|
"completions/mean_length": 54.0654296875,
|
||
|
|
"completions/mean_terminated_length": 54.0654296875,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.1252065480997165,
|
||
|
|
"epoch": 0.0730796586059744,
|
||
|
|
"frac_reward_zero_std": 0.6328125,
|
||
|
|
"grad_norm": 0.8991405367851257,
|
||
|
|
"kl": 0.24130096824632752,
|
||
|
|
"learning_rate": 1.7721338446869976e-07,
|
||
|
|
"loss": 0.0033,
|
||
|
|
"num_tokens": 17973305.0,
|
||
|
|
"reward": 1.189453125,
|
||
|
|
"reward_std": 0.15672777593135834,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1904296875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3925798833370209,
|
||
|
|
"rewards/format_reward_func/mean": 0.9990234375,
|
||
|
|
"rewards/format_reward_func/std": 0.022097086533904076,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.758608877658844,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000474452972412,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5109315365552902,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7791448831558228,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006075352896004915,
|
||
|
|
"step": 274
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00025671912590041757,
|
||
|
|
"clip_ratio/high_mean": 0.00025671912590041757,
|
||
|
|
"clip_ratio/low_mean": 0.0004464233708050516,
|
||
|
|
"clip_ratio/low_min": 0.0004464233708050516,
|
||
|
|
"clip_ratio/region_mean": 0.0007031424967054692,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 53.328125,
|
||
|
|
"completions/mean_terminated_length": 53.328125,
|
||
|
|
"completions/min_length": 33.0,
|
||
|
|
"completions/min_terminated_length": 33.0,
|
||
|
|
"entropy": 0.11954044260912472,
|
||
|
|
"epoch": 0.07361308677098151,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.5922167301177979,
|
||
|
|
"kl": 0.25013600910703343,
|
||
|
|
"learning_rate": 1.7377439408704392e-07,
|
||
|
|
"loss": -0.0024,
|
||
|
|
"num_tokens": 18078657.0,
|
||
|
|
"reward": 1.17578125,
|
||
|
|
"reward_std": 0.11974846571683884,
|
||
|
|
"rewards/equation_reward_func/mean": 0.17578125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3810062110424042,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5618880987167358,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997606873512268,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4763152599334717,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7416753768920898,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005760817788541317,
|
||
|
|
"step": 276
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.727786967530847e-05,
|
||
|
|
"clip_ratio/high_mean": 6.727786967530847e-05,
|
||
|
|
"clip_ratio/low_mean": 9.992629768223398e-05,
|
||
|
|
"clip_ratio/low_min": 9.992629768223398e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00016720416735754244,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 54.8671875,
|
||
|
|
"completions/mean_terminated_length": 54.8671875,
|
||
|
|
"completions/min_length": 33.0,
|
||
|
|
"completions/min_terminated_length": 33.0,
|
||
|
|
"entropy": 0.11932587520115906,
|
||
|
|
"epoch": 0.07414651493598862,
|
||
|
|
"frac_reward_zero_std": 0.765625,
|
||
|
|
"grad_norm": 0.6116552352905273,
|
||
|
|
"kl": 0.19230408842364946,
|
||
|
|
"learning_rate": 1.7035123367668323e-07,
|
||
|
|
"loss": -0.0021,
|
||
|
|
"num_tokens": 18185133.0,
|
||
|
|
"reward": 1.126953125,
|
||
|
|
"reward_std": 0.10245292633771896,
|
||
|
|
"rewards/equation_reward_func/mean": 0.126953125,
|
||
|
|
"rewards/equation_reward_func/std": 0.33324605226516724,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7090688943862915,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999284148216248,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.41860297322273254,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8708324432373047,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006055726669728756,
|
||
|
|
"step": 278
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001274182965668539,
|
||
|
|
"clip_ratio/high_mean": 0.0001274182965668539,
|
||
|
|
"clip_ratio/low_mean": 0.00015632986711959043,
|
||
|
|
"clip_ratio/low_min": 0.00015632986711959043,
|
||
|
|
"clip_ratio/region_mean": 0.00028374816368644434,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 68.0,
|
||
|
|
"completions/max_terminated_length": 68.0,
|
||
|
|
"completions/mean_length": 53.59765625,
|
||
|
|
"completions/mean_terminated_length": 53.59765625,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.10979976960354382,
|
||
|
|
"epoch": 0.07467994310099574,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.7363688349723816,
|
||
|
|
"kl": 0.21388624215291607,
|
||
|
|
"learning_rate": 1.6694461413430893e-07,
|
||
|
|
"loss": 0.0044,
|
||
|
|
"num_tokens": 18290639.0,
|
||
|
|
"reward": 1.18359375,
|
||
|
|
"reward_std": 0.12020279467105865,
|
||
|
|
"rewards/equation_reward_func/mean": 0.18359375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3875311613082886,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6355280876159668,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000213623046875,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5373046398162842,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.621190071105957,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005433195736259222,
|
||
|
|
"step": 280
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012727488905915784,
|
||
|
|
"clip_ratio/high_mean": 0.00012727488905915784,
|
||
|
|
"clip_ratio/low_mean": 0.00022515782297381925,
|
||
|
|
"clip_ratio/low_min": 0.00022515782297381925,
|
||
|
|
"clip_ratio/region_mean": 0.0003524327120329771,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 73.0,
|
||
|
|
"completions/max_terminated_length": 73.0,
|
||
|
|
"completions/mean_length": 54.451171875,
|
||
|
|
"completions/mean_terminated_length": 54.451171875,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11184248907698525,
|
||
|
|
"epoch": 0.07521337126600285,
|
||
|
|
"frac_reward_zero_std": 0.625,
|
||
|
|
"grad_norm": 0.7730035185813904,
|
||
|
|
"kl": 0.24625489198499256,
|
||
|
|
"learning_rate": 1.6355524292152684e-07,
|
||
|
|
"loss": -0.0023,
|
||
|
|
"num_tokens": 18396798.0,
|
||
|
|
"reward": 1.16796875,
|
||
|
|
"reward_std": 0.16813695430755615,
|
||
|
|
"rewards/equation_reward_func/mean": 0.169921875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3759314715862274,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.709104061126709,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000386238098145,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.48130878806114197,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7312462329864502,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005882616620510817,
|
||
|
|
"step": 282
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00022572680609300733,
|
||
|
|
"clip_ratio/high_mean": 0.00022572680609300733,
|
||
|
|
"clip_ratio/low_mean": 0.0002221358267383443,
|
||
|
|
"clip_ratio/low_min": 0.0002221358267383443,
|
||
|
|
"clip_ratio/region_mean": 0.0004478626295975927,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 119.0,
|
||
|
|
"completions/max_terminated_length": 119.0,
|
||
|
|
"completions/mean_length": 53.83203125,
|
||
|
|
"completions/mean_terminated_length": 53.83203125,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11374843782848781,
|
||
|
|
"epoch": 0.07574679943100995,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 0.7327376008033752,
|
||
|
|
"kl": 0.2505280135406388,
|
||
|
|
"learning_rate": 1.6018382391793722e-07,
|
||
|
|
"loss": 0.0008,
|
||
|
|
"num_tokens": 18502408.0,
|
||
|
|
"reward": 1.1171875,
|
||
|
|
"reward_std": 0.1536710113286972,
|
||
|
|
"rewards/equation_reward_func/mean": 0.119140625,
|
||
|
|
"rewards/equation_reward_func/std": 0.32427072525024414,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6212329864501953,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998156428337097,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47825267910957336,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7376160621643066,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005457405000925064,
|
||
|
|
"step": 284
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00019255069976982972,
|
||
|
|
"clip_ratio/high_mean": 0.00019255069976982972,
|
||
|
|
"clip_ratio/low_mean": 0.0001325318201755484,
|
||
|
|
"clip_ratio/low_min": 0.0001325318201755484,
|
||
|
|
"clip_ratio/region_mean": 0.0003250825199453781,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 68.0,
|
||
|
|
"completions/max_terminated_length": 68.0,
|
||
|
|
"completions/mean_length": 53.380859375,
|
||
|
|
"completions/mean_terminated_length": 53.380859375,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.11636582741306888,
|
||
|
|
"epoch": 0.07628022759601708,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.6755468845367432,
|
||
|
|
"kl": 0.3760374633388387,
|
||
|
|
"learning_rate": 1.5683105727495778e-07,
|
||
|
|
"loss": -0.0023,
|
||
|
|
"num_tokens": 18607803.0,
|
||
|
|
"reward": 1.12890625,
|
||
|
|
"reward_std": 0.12402082234621048,
|
||
|
|
"rewards/equation_reward_func/mean": 0.12890625,
|
||
|
|
"rewards/equation_reward_func/std": 0.33542385697364807,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.724694013595581,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000168085098267,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6067004799842834,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5450496673583984,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0060112448409199715,
|
||
|
|
"step": 286
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00016007068269472156,
|
||
|
|
"clip_ratio/high_mean": 0.00016007068269472156,
|
||
|
|
"clip_ratio/low_mean": 0.00012752274051308632,
|
||
|
|
"clip_ratio/low_min": 0.00012752274051308632,
|
||
|
|
"clip_ratio/region_mean": 0.0002875934232078079,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 82.0,
|
||
|
|
"completions/max_terminated_length": 82.0,
|
||
|
|
"completions/mean_length": 54.20703125,
|
||
|
|
"completions/mean_terminated_length": 54.20703125,
|
||
|
|
"completions/min_length": 31.0,
|
||
|
|
"completions/min_terminated_length": 31.0,
|
||
|
|
"entropy": 0.1160916495654318,
|
||
|
|
"epoch": 0.07681365576102418,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.6371228694915771,
|
||
|
|
"kl": 0.20908190992971262,
|
||
|
|
"learning_rate": 1.5349763927042168e-07,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 18713781.0,
|
||
|
|
"reward": 1.140625,
|
||
|
|
"reward_std": 0.14104920625686646,
|
||
|
|
"rewards/equation_reward_func/mean": 0.140625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3479743003845215,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.633782982826233,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000022530555725,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4739110767841339,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7467355728149414,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006055853795260191,
|
||
|
|
"step": 288
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.467687651618487e-05,
|
||
|
|
"clip_ratio/high_mean": 6.467687651618487e-05,
|
||
|
|
"clip_ratio/low_mean": 6.382815384616454e-05,
|
||
|
|
"clip_ratio/low_min": 6.382815384616454e-05,
|
||
|
|
"clip_ratio/region_mean": 0.0001285050303623494,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 82.5,
|
||
|
|
"completions/max_terminated_length": 82.5,
|
||
|
|
"completions/mean_length": 53.6474609375,
|
||
|
|
"completions/mean_terminated_length": 53.6474609375,
|
||
|
|
"completions/min_length": 33.5,
|
||
|
|
"completions/min_terminated_length": 33.5,
|
||
|
|
"entropy": 0.1184024533463849,
|
||
|
|
"epoch": 0.07734708392603129,
|
||
|
|
"frac_reward_zero_std": 0.625,
|
||
|
|
"grad_norm": 0.832065761089325,
|
||
|
|
"kl": 0.22019675746560097,
|
||
|
|
"learning_rate": 1.501842621639796e-07,
|
||
|
|
"loss": 0.0077,
|
||
|
|
"num_tokens": 18924860.0,
|
||
|
|
"reward": 1.1875,
|
||
|
|
"reward_std": 0.16658930480480194,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1875,
|
||
|
|
"rewards/equation_reward_func/std": 0.39066198468208313,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4917330741882324,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000223815441132,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.48453810811042786,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7312403917312622,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006352462340146303,
|
||
|
|
"step": 290
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00013053972765596377,
|
||
|
|
"clip_ratio/high_mean": 0.00013053972765596377,
|
||
|
|
"clip_ratio/low_mean": 0.0002916623747700618,
|
||
|
|
"clip_ratio/low_min": 0.0002916623747700618,
|
||
|
|
"clip_ratio/region_mean": 0.0004222021024260256,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 75.0,
|
||
|
|
"completions/max_terminated_length": 75.0,
|
||
|
|
"completions/mean_length": 54.10546875,
|
||
|
|
"completions/mean_terminated_length": 54.10546875,
|
||
|
|
"completions/min_length": 39.0,
|
||
|
|
"completions/min_terminated_length": 39.0,
|
||
|
|
"entropy": 0.12058492480880684,
|
||
|
|
"epoch": 0.07788051209103841,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 1.1296939849853516,
|
||
|
|
"kl": 0.2924755515737666,
|
||
|
|
"learning_rate": 1.4689161405333652e-07,
|
||
|
|
"loss": -0.0042,
|
||
|
|
"num_tokens": 19030778.0,
|
||
|
|
"reward": 1.1640625,
|
||
|
|
"reward_std": 0.16629283130168915,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1640625,
|
||
|
|
"rewards/equation_reward_func/std": 0.37069445848464966,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4458497762680054,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.999947726726532,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47323113679885864,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.748171329498291,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005906645208597183,
|
||
|
|
"step": 292
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00015977974787044028,
|
||
|
|
"clip_ratio/high_mean": 0.00015977974787044028,
|
||
|
|
"clip_ratio/low_mean": 0.0003193130736084034,
|
||
|
|
"clip_ratio/low_min": 0.0003193130736084034,
|
||
|
|
"clip_ratio/region_mean": 0.0004790928214788437,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 73.0,
|
||
|
|
"completions/max_terminated_length": 73.0,
|
||
|
|
"completions/mean_length": 52.966796875,
|
||
|
|
"completions/mean_terminated_length": 52.966796875,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.11835546274152067,
|
||
|
|
"epoch": 0.07841394025604552,
|
||
|
|
"frac_reward_zero_std": 0.625,
|
||
|
|
"grad_norm": 0.6429219245910645,
|
||
|
|
"kl": 0.29682688580618966,
|
||
|
|
"learning_rate": 1.4362037873135255e-07,
|
||
|
|
"loss": 0.0076,
|
||
|
|
"num_tokens": 19135753.0,
|
||
|
|
"reward": 1.26171875,
|
||
|
|
"reward_std": 0.171815887093544,
|
||
|
|
"rewards/equation_reward_func/mean": 0.263671875,
|
||
|
|
"rewards/equation_reward_func/std": 0.4410543739795685,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5525758266448975,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000226616859436,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5377941727638245,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6202793121337891,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005819721147418022,
|
||
|
|
"step": 294
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001345365469913102,
|
||
|
|
"clip_ratio/high_mean": 0.0001345365469913102,
|
||
|
|
"clip_ratio/low_mean": 0.0001629980009359618,
|
||
|
|
"clip_ratio/low_min": 0.0001629980009359618,
|
||
|
|
"clip_ratio/region_mean": 0.000297534547927272,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 72.0,
|
||
|
|
"completions/max_terminated_length": 72.0,
|
||
|
|
"completions/mean_length": 53.865234375,
|
||
|
|
"completions/mean_terminated_length": 53.865234375,
|
||
|
|
"completions/min_length": 28.0,
|
||
|
|
"completions/min_terminated_length": 28.0,
|
||
|
|
"entropy": 0.11495288068221675,
|
||
|
|
"epoch": 0.07894736842105263,
|
||
|
|
"frac_reward_zero_std": 0.625,
|
||
|
|
"grad_norm": 0.5954465270042419,
|
||
|
|
"kl": 0.24669315500391853,
|
||
|
|
"learning_rate": 1.403712355440378e-07,
|
||
|
|
"loss": -0.0138,
|
||
|
|
"num_tokens": 19241508.0,
|
||
|
|
"reward": 1.134765625,
|
||
|
|
"reward_std": 0.15756143629550934,
|
||
|
|
"rewards/equation_reward_func/mean": 0.134765625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3418070077896118,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4350379705429077,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998484253883362,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4763599932193756,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7415814399719238,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005357466638088226,
|
||
|
|
"step": 296
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012767812909765376,
|
||
|
|
"clip_ratio/high_mean": 0.00012767812909765376,
|
||
|
|
"clip_ratio/low_mean": 0.0001937980479043391,
|
||
|
|
"clip_ratio/low_min": 0.0001937980479043391,
|
||
|
|
"clip_ratio/region_mean": 0.0003214761770019929,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 81.0,
|
||
|
|
"completions/max_terminated_length": 81.0,
|
||
|
|
"completions/mean_length": 54.123046875,
|
||
|
|
"completions/mean_terminated_length": 54.123046875,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.10710652855535348,
|
||
|
|
"epoch": 0.07948079658605975,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.5267983675003052,
|
||
|
|
"kl": 0.24504679979549515,
|
||
|
|
"learning_rate": 1.371448592494707e-07,
|
||
|
|
"loss": 0.0166,
|
||
|
|
"num_tokens": 19347403.0,
|
||
|
|
"reward": 1.123046875,
|
||
|
|
"reward_std": 0.13413512706756592,
|
||
|
|
"rewards/equation_reward_func/mean": 0.125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3310423493385315,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4578505754470825,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998474717140198,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6094253659248352,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4952387809753418,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005574563052505255,
|
||
|
|
"step": 298
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.526715600966579e-05,
|
||
|
|
"clip_ratio/high_mean": 9.526715600966579e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00012599391953295303,
|
||
|
|
"clip_ratio/low_min": 0.00012599391953295303,
|
||
|
|
"clip_ratio/region_mean": 0.00022126107554261884,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 87.0,
|
||
|
|
"completions/max_terminated_length": 87.0,
|
||
|
|
"completions/mean_length": 54.29296875,
|
||
|
|
"completions/mean_terminated_length": 54.29296875,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.10549763672881657,
|
||
|
|
"epoch": 0.08001422475106686,
|
||
|
|
"frac_reward_zero_std": 0.609375,
|
||
|
|
"grad_norm": 0.5649374723434448,
|
||
|
|
"kl": 0.20756946214371258,
|
||
|
|
"learning_rate": 1.3394191987766996e-07,
|
||
|
|
"loss": -0.0085,
|
||
|
|
"num_tokens": 19453401.0,
|
||
|
|
"reward": 1.154296875,
|
||
|
|
"reward_std": 0.17846155166625977,
|
||
|
|
"rewards/equation_reward_func/mean": 0.154296875,
|
||
|
|
"rewards/equation_reward_func/std": 0.36158639192581177,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.9475939273834229,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.999893069267273,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6712517738342285,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6665947437286377,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005600410513579845,
|
||
|
|
"step": 300
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00016323415265004669,
|
||
|
|
"clip_ratio/high_mean": 0.00016323415265004669,
|
||
|
|
"clip_ratio/low_mean": 0.0002246858881941686,
|
||
|
|
"clip_ratio/low_min": 0.0002246858881941686,
|
||
|
|
"clip_ratio/region_mean": 0.0003879200408442153,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 71.0,
|
||
|
|
"completions/max_terminated_length": 71.0,
|
||
|
|
"completions/mean_length": 53.71875,
|
||
|
|
"completions/mean_terminated_length": 53.71875,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.10518266074359417,
|
||
|
|
"epoch": 0.08054765291607396,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.6784906387329102,
|
||
|
|
"kl": 0.483879576333695,
|
||
|
|
"learning_rate": 1.3076308259144652e-07,
|
||
|
|
"loss": 0.0048,
|
||
|
|
"num_tokens": 19558953.0,
|
||
|
|
"reward": 1.1796875,
|
||
|
|
"reward_std": 0.12743577361106873,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1796875,
|
||
|
|
"rewards/equation_reward_func/std": 0.38430243730545044,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.575277328491211,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999927878379822,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5361084342002869,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6234188079833984,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005135941784828901,
|
||
|
|
"step": 302
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00013082856700445214,
|
||
|
|
"clip_ratio/high_mean": 0.00013082856700445214,
|
||
|
|
"clip_ratio/low_mean": 0.00040843233970614773,
|
||
|
|
"clip_ratio/low_min": 0.00040843233970614773,
|
||
|
|
"clip_ratio/region_mean": 0.0005392609067105999,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 53.46875,
|
||
|
|
"completions/mean_terminated_length": 53.46875,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.105962166769637,
|
||
|
|
"epoch": 0.08108108108108109,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 0.5560368299484253,
|
||
|
|
"kl": 0.23596913264029556,
|
||
|
|
"learning_rate": 1.2760900754826858e-07,
|
||
|
|
"loss": -0.0004,
|
||
|
|
"num_tokens": 19664321.0,
|
||
|
|
"reward": 1.1953125,
|
||
|
|
"reward_std": 0.1626114547252655,
|
||
|
|
"rewards/equation_reward_func/mean": 0.197265625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3983237147331238,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5302678346633911,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.999779224395752,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.3679559528827667,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.9997920989990234,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005438545718789101,
|
||
|
|
"step": 304
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.4083162770710056e-05,
|
||
|
|
"clip_ratio/high_mean": 3.4083162770710056e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0002949693912847175,
|
||
|
|
"clip_ratio/low_min": 0.0002949693912847175,
|
||
|
|
"clip_ratio/region_mean": 0.0003290525540554275,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 80.5,
|
||
|
|
"completions/max_terminated_length": 80.5,
|
||
|
|
"completions/mean_length": 53.685546875,
|
||
|
|
"completions/mean_terminated_length": 53.685546875,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.10773880945311652,
|
||
|
|
"epoch": 0.0816145092460882,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.6665006875991821,
|
||
|
|
"kl": 0.3991919548975097,
|
||
|
|
"learning_rate": 1.2448034976316394e-07,
|
||
|
|
"loss": 0.0043,
|
||
|
|
"num_tokens": 19875319.0,
|
||
|
|
"reward": 1.171875,
|
||
|
|
"reward_std": 0.13664264231920242,
|
||
|
|
"rewards/equation_reward_func/mean": 0.171875,
|
||
|
|
"rewards/equation_reward_func/std": 0.37562617659568787,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5477519631385803,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001848340034485,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5895287990570068,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5308138132095337,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005096454173326492,
|
||
|
|
"step": 306
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.455278425063525e-05,
|
||
|
|
"clip_ratio/high_mean": 6.455278425063525e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0001954515884992563,
|
||
|
|
"clip_ratio/low_min": 0.0001954515884992563,
|
||
|
|
"clip_ratio/region_mean": 0.00026000437274989154,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 53.947265625,
|
||
|
|
"completions/mean_terminated_length": 53.947265625,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.10061582881543371,
|
||
|
|
"epoch": 0.0821479374110953,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.5578693151473999,
|
||
|
|
"kl": 0.26390944690340096,
|
||
|
|
"learning_rate": 1.213777589726922e-07,
|
||
|
|
"loss": 0.0015,
|
||
|
|
"num_tokens": 19981004.0,
|
||
|
|
"reward": 1.162109375,
|
||
|
|
"reward_std": 0.1520920991897583,
|
||
|
|
"rewards/equation_reward_func/mean": 0.162109375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3689115643501282,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6302008628845215,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9996868371963501,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47551459074020386,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7433576583862305,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005319277290254831,
|
||
|
|
"step": 308
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00013303185632038448,
|
||
|
|
"clip_ratio/high_mean": 0.00013303185632038448,
|
||
|
|
"clip_ratio/low_mean": 9.891274385154247e-05,
|
||
|
|
"clip_ratio/low_min": 9.891274385154247e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00023194460017192695,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 115.0,
|
||
|
|
"completions/max_terminated_length": 115.0,
|
||
|
|
"completions/mean_length": 54.548828125,
|
||
|
|
"completions/mean_terminated_length": 54.548828125,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.10588591939045323,
|
||
|
|
"epoch": 0.08268136557610242,
|
||
|
|
"frac_reward_zero_std": 0.625,
|
||
|
|
"grad_norm": 0.7521274089813232,
|
||
|
|
"kl": 0.27938652514583534,
|
||
|
|
"learning_rate": 1.183018795000118e-07,
|
||
|
|
"loss": -0.0094,
|
||
|
|
"num_tokens": 20087141.0,
|
||
|
|
"reward": 1.201171875,
|
||
|
|
"reward_std": 0.16781118512153625,
|
||
|
|
"rewards/equation_reward_func/mean": 0.201171875,
|
||
|
|
"rewards/equation_reward_func/std": 0.4012683033943176,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5131198167800903,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.999969482421875,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5128318667411804,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6678072214126587,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005433302838355303,
|
||
|
|
"step": 310
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.650004600795607e-05,
|
||
|
|
"clip_ratio/high_mean": 9.650004600795607e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0002817239728756249,
|
||
|
|
"clip_ratio/low_min": 0.0002817239728756249,
|
||
|
|
"clip_ratio/region_mean": 0.000378224018883581,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 54.435546875,
|
||
|
|
"completions/mean_terminated_length": 54.435546875,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.10187006679673989,
|
||
|
|
"epoch": 0.08321479374110953,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.49840039014816284,
|
||
|
|
"kl": 0.21569028579526478,
|
||
|
|
"learning_rate": 1.1525335012107188e-07,
|
||
|
|
"loss": 0.002,
|
||
|
|
"num_tokens": 20193292.0,
|
||
|
|
"reward": 1.076171875,
|
||
|
|
"reward_std": 0.10514955222606659,
|
||
|
|
"rewards/equation_reward_func/mean": 0.076171875,
|
||
|
|
"rewards/equation_reward_func/std": 0.26553234457969666,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6213353872299194,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.999853789806366,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5379842519760132,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6199259757995605,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.00533775519579649,
|
||
|
|
"step": 312
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012695331436892351,
|
||
|
|
"clip_ratio/high_mean": 0.00012695331436892351,
|
||
|
|
"clip_ratio/low_mean": 0.00022922881503796412,
|
||
|
|
"clip_ratio/low_min": 0.00022922881503796412,
|
||
|
|
"clip_ratio/region_mean": 0.00035618212940688763,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 53.759765625,
|
||
|
|
"completions/mean_terminated_length": 53.759765625,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.09872452769842413,
|
||
|
|
"epoch": 0.08374822190611664,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.8471410870552063,
|
||
|
|
"kl": 0.24236748119195303,
|
||
|
|
"learning_rate": 1.1223280393195566e-07,
|
||
|
|
"loss": 0.0079,
|
||
|
|
"num_tokens": 20298905.0,
|
||
|
|
"reward": 1.138671875,
|
||
|
|
"reward_std": 0.14611473679542542,
|
||
|
|
"rewards/equation_reward_func/mean": 0.138671875,
|
||
|
|
"rewards/equation_reward_func/std": 0.34594178199768066,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8655023574829102,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001434087753296,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6093878746032715,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.623530387878418,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0052959248423576355,
|
||
|
|
"step": 314
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00019498604039351145,
|
||
|
|
"clip_ratio/high_mean": 0.00019498604039351145,
|
||
|
|
"clip_ratio/low_mean": 0.00022457260961851312,
|
||
|
|
"clip_ratio/low_min": 0.00022457260961851312,
|
||
|
|
"clip_ratio/region_mean": 0.00041955865001202456,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 78.0,
|
||
|
|
"completions/max_terminated_length": 78.0,
|
||
|
|
"completions/mean_length": 52.978515625,
|
||
|
|
"completions/mean_terminated_length": 52.978515625,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.09926377278235224,
|
||
|
|
"epoch": 0.08428165007112376,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.5593821406364441,
|
||
|
|
"kl": 0.26706351588169736,
|
||
|
|
"learning_rate": 1.0924086821740436e-07,
|
||
|
|
"loss": -0.0012,
|
||
|
|
"num_tokens": 20403910.0,
|
||
|
|
"reward": 1.158203125,
|
||
|
|
"reward_std": 0.12848231196403503,
|
||
|
|
"rewards/equation_reward_func/mean": 0.158203125,
|
||
|
|
"rewards/equation_reward_func/std": 0.36528825759887695,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.3883647918701172,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999933242797852,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5356259942054749,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6243190765380859,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.004806127864867449,
|
||
|
|
"step": 316
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00019597292864798673,
|
||
|
|
"clip_ratio/high_mean": 0.00019597292864798673,
|
||
|
|
"clip_ratio/low_mean": 0.00016192727748097645,
|
||
|
|
"clip_ratio/low_min": 0.00016192727748097645,
|
||
|
|
"clip_ratio/region_mean": 0.00035790020612896315,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 53.642578125,
|
||
|
|
"completions/mean_terminated_length": 53.642578125,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.10184383454422156,
|
||
|
|
"epoch": 0.08481507823613087,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 0.5862502455711365,
|
||
|
|
"kl": 0.27017481583687997,
|
||
|
|
"learning_rate": 1.0627816432054689e-07,
|
||
|
|
"loss": -0.004,
|
||
|
|
"num_tokens": 20509407.0,
|
||
|
|
"reward": 1.177734375,
|
||
|
|
"reward_std": 0.1521681845188141,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1796875,
|
||
|
|
"rewards/equation_reward_func/std": 0.38430243730545044,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8215539455413818,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000892877578735,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5373824834823608,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6210451126098633,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.004881645552814007,
|
||
|
|
"step": 318
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00016617278258005777,
|
||
|
|
"clip_ratio/high_mean": 0.00016617278258005777,
|
||
|
|
"clip_ratio/low_mean": 3.062599636096921e-05,
|
||
|
|
"clip_ratio/low_min": 3.062599636096921e-05,
|
||
|
|
"clip_ratio/region_mean": 0.000196798778941027,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 53.58984375,
|
||
|
|
"completions/mean_terminated_length": 53.58984375,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.09652745206322935,
|
||
|
|
"epoch": 0.08534850640113797,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.7487227916717529,
|
||
|
|
"kl": 0.30668989858693546,
|
||
|
|
"learning_rate": 1.0334530751386386e-07,
|
||
|
|
"loss": 0.0024,
|
||
|
|
"num_tokens": 20614861.0,
|
||
|
|
"reward": 1.14453125,
|
||
|
|
"reward_std": 0.13058918714523315,
|
||
|
|
"rewards/equation_reward_func/mean": 0.14453125,
|
||
|
|
"rewards/equation_reward_func/std": 0.35197147727012634,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.9990084171295166,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999464154243469,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47329431772232056,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7480378150939941,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.004811522550880909,
|
||
|
|
"step": 320
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 2.890507716478573e-05,
|
||
|
|
"clip_ratio/high_mean": 2.890507716478573e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00012445226376358833,
|
||
|
|
"clip_ratio/low_min": 0.00012445226376358833,
|
||
|
|
"clip_ratio/region_mean": 0.00015335734092837406,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 85.0,
|
||
|
|
"completions/max_terminated_length": 85.0,
|
||
|
|
"completions/mean_length": 54.3740234375,
|
||
|
|
"completions/mean_terminated_length": 54.3740234375,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.09888760331604216,
|
||
|
|
"epoch": 0.0858819345661451,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.5923455357551575,
|
||
|
|
"kl": 0.288042470605837,
|
||
|
|
"learning_rate": 1.0044290687141255e-07,
|
||
|
|
"loss": -0.0038,
|
||
|
|
"num_tokens": 20826892.0,
|
||
|
|
"reward": 1.1435546875,
|
||
|
|
"reward_std": 0.12148462608456612,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1435546875,
|
||
|
|
"rewards/equation_reward_func/std": 0.34574414789676666,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.589076280593872,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000116765499115,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.34139294922351837,
|
||
|
|
"sampling/sampling_logp_difference/max": 1.1013591885566711,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.004871859215199947,
|
||
|
|
"step": 322
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00016105886445277266,
|
||
|
|
"clip_ratio/high_mean": 0.00016105886445277266,
|
||
|
|
"clip_ratio/low_mean": 0.0001920885891498377,
|
||
|
|
"clip_ratio/low_min": 0.0001920885891498377,
|
||
|
|
"clip_ratio/region_mean": 0.0003531474536026104,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 54.263671875,
|
||
|
|
"completions/mean_terminated_length": 54.263671875,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.09685155438880126,
|
||
|
|
"epoch": 0.0864153627311522,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.4126549959182739,
|
||
|
|
"kl": 0.2570730716817909,
|
||
|
|
"learning_rate": 9.757156514233892e-08,
|
||
|
|
"loss": 0.0147,
|
||
|
|
"num_tokens": 20932923.0,
|
||
|
|
"reward": 1.126953125,
|
||
|
|
"reward_std": 0.13776035606861115,
|
||
|
|
"rewards/equation_reward_func/mean": 0.126953125,
|
||
|
|
"rewards/equation_reward_func/std": 0.33324605226516724,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5707911252975464,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0002734661102295,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4189390242099762,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8700299263000488,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005099819041788578,
|
||
|
|
"step": 324
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012838508700951934,
|
||
|
|
"clip_ratio/high_mean": 0.00012838508700951934,
|
||
|
|
"clip_ratio/low_mean": 9.717278104896347e-05,
|
||
|
|
"clip_ratio/low_min": 9.717278104896347e-05,
|
||
|
|
"clip_ratio/region_mean": 0.0002255578680584828,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 69.0,
|
||
|
|
"completions/max_terminated_length": 69.0,
|
||
|
|
"completions/mean_length": 53.513671875,
|
||
|
|
"completions/mean_terminated_length": 53.513671875,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.09719797016845809,
|
||
|
|
"epoch": 0.08694879089615931,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.6993542909622192,
|
||
|
|
"kl": 0.22395568154752254,
|
||
|
|
"learning_rate": 9.473187862570289e-08,
|
||
|
|
"loss": -0.0121,
|
||
|
|
"num_tokens": 21038298.0,
|
||
|
|
"reward": 1.20703125,
|
||
|
|
"reward_std": 0.13973549008369446,
|
||
|
|
"rewards/equation_reward_func/mean": 0.20703125,
|
||
|
|
"rewards/equation_reward_func/std": 0.40557438135147095,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5042448043823242,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001046657562256,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5376067757606506,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6206278800964355,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.004805511329323053,
|
||
|
|
"step": 326
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00013057675096206367,
|
||
|
|
"clip_ratio/high_mean": 0.00013057675096206367,
|
||
|
|
"clip_ratio/low_mean": 6.622885444408489e-05,
|
||
|
|
"clip_ratio/low_min": 6.622885444408489e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00019680560540614856,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 72.0,
|
||
|
|
"completions/max_terminated_length": 72.0,
|
||
|
|
"completions/mean_length": 53.38671875,
|
||
|
|
"completions/mean_terminated_length": 53.38671875,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.09837449114355776,
|
||
|
|
"epoch": 0.08748221906116643,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.7079157829284668,
|
||
|
|
"kl": 0.1822098141743077,
|
||
|
|
"learning_rate": 9.192443704664344e-08,
|
||
|
|
"loss": 0.0066,
|
||
|
|
"num_tokens": 21143592.0,
|
||
|
|
"reward": 1.18359375,
|
||
|
|
"reward_std": 0.15097439289093018,
|
||
|
|
"rewards/equation_reward_func/mean": 0.185546875,
|
||
|
|
"rewards/equation_reward_func/std": 0.38912075757980347,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5526940822601318,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000152349472046,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6077161431312561,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4980473518371582,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.00505709508433938,
|
||
|
|
"step": 328
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00019603894260298047,
|
||
|
|
"clip_ratio/high_mean": 0.00019603894260298047,
|
||
|
|
"clip_ratio/low_mean": 0.00019100008648820221,
|
||
|
|
"clip_ratio/low_min": 0.00019100008648820221,
|
||
|
|
"clip_ratio/region_mean": 0.0003870390290911827,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 71.0,
|
||
|
|
"completions/max_terminated_length": 71.0,
|
||
|
|
"completions/mean_length": 54.005859375,
|
||
|
|
"completions/mean_terminated_length": 54.005859375,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.10252868135770161,
|
||
|
|
"epoch": 0.08801564722617354,
|
||
|
|
"frac_reward_zero_std": 0.765625,
|
||
|
|
"grad_norm": 0.4996802508831024,
|
||
|
|
"kl": 0.28261983104878,
|
||
|
|
"learning_rate": 8.914982343390895e-08,
|
||
|
|
"loss": -0.0071,
|
||
|
|
"num_tokens": 21249315.0,
|
||
|
|
"reward": 1.08984375,
|
||
|
|
"reward_std": 0.10902006179094315,
|
||
|
|
"rewards/equation_reward_func/mean": 0.08984375,
|
||
|
|
"rewards/equation_reward_func/std": 0.2862374484539032,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6295589208602905,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999970197677612,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5906658172607422,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5265048742294312,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005076720379292965,
|
||
|
|
"step": 330
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.288123808594214e-05,
|
||
|
|
"clip_ratio/high_mean": 6.288123808594214e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00012890514658970965,
|
||
|
|
"clip_ratio/low_min": 0.00012890514658970965,
|
||
|
|
"clip_ratio/region_mean": 0.0001917863846756518,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 53.537109375,
|
||
|
|
"completions/mean_terminated_length": 53.537109375,
|
||
|
|
"completions/min_length": 31.0,
|
||
|
|
"completions/min_terminated_length": 31.0,
|
||
|
|
"entropy": 0.09989326985345946,
|
||
|
|
"epoch": 0.08854907539118065,
|
||
|
|
"frac_reward_zero_std": 0.609375,
|
||
|
|
"grad_norm": 0.7519966959953308,
|
||
|
|
"kl": 0.21807977060476938,
|
||
|
|
"learning_rate": 8.640861399877805e-08,
|
||
|
|
"loss": 0.007,
|
||
|
|
"num_tokens": 21354734.0,
|
||
|
|
"reward": 1.22265625,
|
||
|
|
"reward_std": 0.1729392409324646,
|
||
|
|
"rewards/equation_reward_func/mean": 0.224609375,
|
||
|
|
"rewards/equation_reward_func/std": 0.41773295402526855,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5419955253601074,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000159740447998,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.41770172119140625,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8729877471923828,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005091718398034573,
|
||
|
|
"step": 332
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.233734363069137e-05,
|
||
|
|
"clip_ratio/high_mean": 3.233734363069137e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00038376119401719834,
|
||
|
|
"clip_ratio/low_min": 0.00038376119401719834,
|
||
|
|
"clip_ratio/region_mean": 0.0004160985376478897,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 100.0,
|
||
|
|
"completions/max_terminated_length": 100.0,
|
||
|
|
"completions/mean_length": 54.75390625,
|
||
|
|
"completions/mean_terminated_length": 54.75390625,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.09954109022186862,
|
||
|
|
"epoch": 0.08908250355618777,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.6240397691726685,
|
||
|
|
"kl": 0.21472362304727236,
|
||
|
|
"learning_rate": 8.370137801539634e-08,
|
||
|
|
"loss": 0.0042,
|
||
|
|
"num_tokens": 21461064.0,
|
||
|
|
"reward": 1.138671875,
|
||
|
|
"reward_std": 0.14157837629318237,
|
||
|
|
"rewards/equation_reward_func/mean": 0.138671875,
|
||
|
|
"rewards/equation_reward_func/std": 0.34594178199768066,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5789878368377686,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0002344846725464,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4178570806980133,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8726158142089844,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005193983670324087,
|
||
|
|
"step": 334
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.585934458300471e-05,
|
||
|
|
"clip_ratio/high_mean": 9.585934458300471e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00012715121071475247,
|
||
|
|
"clip_ratio/low_min": 0.00012715121071475247,
|
||
|
|
"clip_ratio/region_mean": 0.00022301055529775718,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 70.0,
|
||
|
|
"completions/max_terminated_length": 70.0,
|
||
|
|
"completions/mean_length": 54.646484375,
|
||
|
|
"completions/mean_terminated_length": 54.646484375,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.10077821670307054,
|
||
|
|
"epoch": 0.08961593172119488,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.6173285245895386,
|
||
|
|
"kl": 0.2103740041040712,
|
||
|
|
"learning_rate": 8.102867770255337e-08,
|
||
|
|
"loss": -0.0037,
|
||
|
|
"num_tokens": 21567323.0,
|
||
|
|
"reward": 1.140625,
|
||
|
|
"reward_std": 0.1521514505147934,
|
||
|
|
"rewards/equation_reward_func/mean": 0.140625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3479743003845215,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6297080516815186,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000563859939575,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5369880199432373,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6217794418334961,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0048846774734556675,
|
||
|
|
"step": 336
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.306878433148894e-05,
|
||
|
|
"clip_ratio/high_mean": 3.306878433148894e-05,
|
||
|
|
"clip_ratio/low_mean": 6.596661599663396e-05,
|
||
|
|
"clip_ratio/low_min": 6.596661599663396e-05,
|
||
|
|
"clip_ratio/region_mean": 9.903540032812291e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 78.0,
|
||
|
|
"completions/max_terminated_length": 78.0,
|
||
|
|
"completions/mean_length": 53.82421875,
|
||
|
|
"completions/mean_terminated_length": 53.82421875,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.09935227346916993,
|
||
|
|
"epoch": 0.09014935988620199,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.6280437707901001,
|
||
|
|
"kl": 0.2277641739282343,
|
||
|
|
"learning_rate": 7.839106810692589e-08,
|
||
|
|
"loss": -0.0009,
|
||
|
|
"num_tokens": 21778639.0,
|
||
|
|
"reward": 1.1806640625,
|
||
|
|
"reward_std": 0.13480134308338165,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1806640625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3841889351606369,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7198472619056702,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999500811100006,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6093293130397797,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.613457202911377,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.004906615940853953,
|
||
|
|
"step": 338
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.600156206741101e-05,
|
||
|
|
"clip_ratio/high_mean": 9.600156206741101e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00012920183427114453,
|
||
|
|
"clip_ratio/low_min": 0.00012920183427114453,
|
||
|
|
"clip_ratio/region_mean": 0.00022520339633855555,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 77.0,
|
||
|
|
"completions/max_terminated_length": 77.0,
|
||
|
|
"completions/mean_length": 54.6484375,
|
||
|
|
"completions/mean_terminated_length": 54.6484375,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.09617070563965374,
|
||
|
|
"epoch": 0.0906827880512091,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.7513833045959473,
|
||
|
|
"kl": 0.42414102144539356,
|
||
|
|
"learning_rate": 7.57890969878093e-08,
|
||
|
|
"loss": -0.0044,
|
||
|
|
"num_tokens": 21884891.0,
|
||
|
|
"reward": 1.21875,
|
||
|
|
"reward_std": 0.14459636807441711,
|
||
|
|
"rewards/equation_reward_func/mean": 0.220703125,
|
||
|
|
"rewards/equation_reward_func/std": 0.4151262938976288,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6317570209503174,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000431537628174,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6167507171630859,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4896574020385742,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.004871396813541651,
|
||
|
|
"step": 340
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012448259642244212,
|
||
|
|
"clip_ratio/high_mean": 0.00012448259642244212,
|
||
|
|
"clip_ratio/low_mean": 9.562747759951485e-05,
|
||
|
|
"clip_ratio/low_min": 9.562747759951485e-05,
|
||
|
|
"clip_ratio/region_mean": 0.000220110074021957,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 70.0,
|
||
|
|
"completions/max_terminated_length": 70.0,
|
||
|
|
"completions/mean_length": 53.703125,
|
||
|
|
"completions/mean_terminated_length": 53.703125,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.0994571488764551,
|
||
|
|
"epoch": 0.09121621621621621,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.6678158044815063,
|
||
|
|
"kl": 0.5161584214203887,
|
||
|
|
"learning_rate": 7.322330470336313e-08,
|
||
|
|
"loss": 0.0061,
|
||
|
|
"num_tokens": 21990387.0,
|
||
|
|
"reward": 1.1328125,
|
||
|
|
"reward_std": 0.12927988171577454,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1328125,
|
||
|
|
"rewards/equation_reward_func/std": 0.33970388770103455,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4448696374893188,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999097585678101,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47708994150161743,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7400503158569336,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005025411024689674,
|
||
|
|
"step": 342
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.34941356515305e-05,
|
||
|
|
"clip_ratio/high_mean": 6.34941356515305e-05,
|
||
|
|
"clip_ratio/low_mean": 9.748389452902807e-05,
|
||
|
|
"clip_ratio/low_min": 9.748389452902807e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00016097803018055856,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 80.0,
|
||
|
|
"completions/max_terminated_length": 80.0,
|
||
|
|
"completions/mean_length": 54.388671875,
|
||
|
|
"completions/mean_terminated_length": 54.388671875,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.10490797604951593,
|
||
|
|
"epoch": 0.09174964438122332,
|
||
|
|
"frac_reward_zero_std": 0.609375,
|
||
|
|
"grad_norm": 0.5954409241676331,
|
||
|
|
"kl": 0.23606261528200573,
|
||
|
|
"learning_rate": 7.069422409839363e-08,
|
||
|
|
"loss": 0.0057,
|
||
|
|
"num_tokens": 22096346.0,
|
||
|
|
"reward": 1.205078125,
|
||
|
|
"reward_std": 0.17872552573680878,
|
||
|
|
"rewards/equation_reward_func/mean": 0.205078125,
|
||
|
|
"rewards/equation_reward_func/std": 0.4041535556316376,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4162734746932983,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001779794692993,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5386984348297119,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6185994148254395,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005469147115945816,
|
||
|
|
"step": 344
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001925600388656474,
|
||
|
|
"clip_ratio/high_mean": 0.0001925600388656474,
|
||
|
|
"clip_ratio/low_mean": 0.0002929596521425992,
|
||
|
|
"clip_ratio/low_min": 0.0002929596521425992,
|
||
|
|
"clip_ratio/region_mean": 0.0004855196910082466,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 53.69921875,
|
||
|
|
"completions/mean_terminated_length": 53.69921875,
|
||
|
|
"completions/min_length": 44.0,
|
||
|
|
"completions/min_terminated_length": 44.0,
|
||
|
|
"entropy": 0.10846325796511438,
|
||
|
|
"epoch": 0.09228307254623044,
|
||
|
|
"frac_reward_zero_std": 0.578125,
|
||
|
|
"grad_norm": 0.5828835964202881,
|
||
|
|
"kl": 0.21855774811572498,
|
||
|
|
"learning_rate": 6.820238039369647e-08,
|
||
|
|
"loss": -0.0111,
|
||
|
|
"num_tokens": 22201808.0,
|
||
|
|
"reward": 1.208984375,
|
||
|
|
"reward_std": 0.1736050695180893,
|
||
|
|
"rewards/equation_reward_func/mean": 0.208984375,
|
||
|
|
"rewards/equation_reward_func/std": 0.40698084235191345,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6369379758834839,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999135732650757,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.25863245129585266,
|
||
|
|
"sampling/sampling_logp_difference/max": 1.3523473739624023,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0053120688535273075,
|
||
|
|
"step": 346
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012726225152922174,
|
||
|
|
"clip_ratio/high_mean": 0.00012726225152922174,
|
||
|
|
"clip_ratio/low_mean": 0.0001607260104113569,
|
||
|
|
"clip_ratio/low_min": 0.0001607260104113569,
|
||
|
|
"clip_ratio/region_mean": 0.00028798826194057864,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 54.6484375,
|
||
|
|
"completions/mean_terminated_length": 54.6484375,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.10222871187660429,
|
||
|
|
"epoch": 0.09281650071123755,
|
||
|
|
"frac_reward_zero_std": 0.625,
|
||
|
|
"grad_norm": 0.6385054588317871,
|
||
|
|
"kl": 0.20221902802586555,
|
||
|
|
"learning_rate": 6.574829107698238e-08,
|
||
|
|
"loss": 0.0178,
|
||
|
|
"num_tokens": 22308028.0,
|
||
|
|
"reward": 1.22265625,
|
||
|
|
"reward_std": 0.16951988637447357,
|
||
|
|
"rewards/equation_reward_func/mean": 0.22265625,
|
||
|
|
"rewards/equation_reward_func/std": 0.41643625497817993,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8017233610153198,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999393820762634,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6400265693664551,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5887436866760254,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005080735310912132,
|
||
|
|
"step": 348
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012558145681396127,
|
||
|
|
"clip_ratio/high_mean": 0.00012558145681396127,
|
||
|
|
"clip_ratio/low_mean": 6.365163588068551e-05,
|
||
|
|
"clip_ratio/low_min": 6.365163588068551e-05,
|
||
|
|
"clip_ratio/region_mean": 0.0001892330926946468,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 77.0,
|
||
|
|
"completions/max_terminated_length": 77.0,
|
||
|
|
"completions/mean_length": 53.412109375,
|
||
|
|
"completions/mean_terminated_length": 53.412109375,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.10489507785273923,
|
||
|
|
"epoch": 0.09334992887624466,
|
||
|
|
"frac_reward_zero_std": 0.796875,
|
||
|
|
"grad_norm": 0.49755725264549255,
|
||
|
|
"kl": 0.2944268166191048,
|
||
|
|
"learning_rate": 6.333246579540971e-08,
|
||
|
|
"loss": -0.0153,
|
||
|
|
"num_tokens": 22413359.0,
|
||
|
|
"reward": 1.13671875,
|
||
|
|
"reward_std": 0.0862782895565033,
|
||
|
|
"rewards/equation_reward_func/mean": 0.13671875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3438861668109894,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7254022359848022,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000238060951233,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6082823872566223,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5454602241516113,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0053401063196361065,
|
||
|
|
"step": 350
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.313139520999457e-05,
|
||
|
|
"clip_ratio/high_mean": 6.313139520999457e-05,
|
||
|
|
"clip_ratio/low_mean": 9.715992038966053e-05,
|
||
|
|
"clip_ratio/low_min": 9.715992038966053e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00016029131559965512,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 71.0,
|
||
|
|
"completions/max_terminated_length": 71.0,
|
||
|
|
"completions/mean_length": 53.89453125,
|
||
|
|
"completions/mean_terminated_length": 53.89453125,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.10171776244209872,
|
||
|
|
"epoch": 0.09388335704125178,
|
||
|
|
"frac_reward_zero_std": 0.734375,
|
||
|
|
"grad_norm": 0.5731579661369324,
|
||
|
|
"kl": 0.2143559284094307,
|
||
|
|
"learning_rate": 6.095540624974435e-08,
|
||
|
|
"loss": 0.0051,
|
||
|
|
"num_tokens": 22519017.0,
|
||
|
|
"reward": 1.115234375,
|
||
|
|
"reward_std": 0.11099714785814285,
|
||
|
|
"rewards/equation_reward_func/mean": 0.115234375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3196168541908264,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.3801380395889282,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999234080314636,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47313907742500305,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7483658790588379,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.00496212812140584,
|
||
|
|
"step": 352
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.536028942921096e-05,
|
||
|
|
"clip_ratio/high_mean": 6.536028942921096e-05,
|
||
|
|
"clip_ratio/low_mean": 9.709978540841903e-05,
|
||
|
|
"clip_ratio/low_min": 9.709978540841903e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00016246007483763,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 77.0,
|
||
|
|
"completions/max_terminated_length": 77.0,
|
||
|
|
"completions/mean_length": 54.0927734375,
|
||
|
|
"completions/mean_terminated_length": 54.0927734375,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.10690712990860145,
|
||
|
|
"epoch": 0.09441678520625889,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.5846546292304993,
|
||
|
|
"kl": 0.2801658991310332,
|
||
|
|
"learning_rate": 5.861760609017002e-08,
|
||
|
|
"loss": 0.0067,
|
||
|
|
"num_tokens": 22730616.0,
|
||
|
|
"reward": 1.181640625,
|
||
|
|
"reward_std": 0.14229890704154968,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1826171875,
|
||
|
|
"rewards/equation_reward_func/std": 0.38598138093948364,
|
||
|
|
"rewards/format_reward_func/mean": 0.9990234375,
|
||
|
|
"rewards/format_reward_func/std": 0.022097086533904076,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4574605226516724,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999331831932068,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5064413398504257,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6823024749755859,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.00533159333281219,
|
||
|
|
"step": 354
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00028676756644724973,
|
||
|
|
"clip_ratio/high_mean": 0.00028676756644724973,
|
||
|
|
"clip_ratio/low_mean": 0.0001584082298601667,
|
||
|
|
"clip_ratio/low_min": 0.0001584082298601667,
|
||
|
|
"clip_ratio/region_mean": 0.0004451757963074164,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 80.0,
|
||
|
|
"completions/max_terminated_length": 80.0,
|
||
|
|
"completions/mean_length": 53.71484375,
|
||
|
|
"completions/mean_terminated_length": 53.71484375,
|
||
|
|
"completions/min_length": 42.0,
|
||
|
|
"completions/min_terminated_length": 42.0,
|
||
|
|
"entropy": 0.10310176987614897,
|
||
|
|
"epoch": 0.09495021337126601,
|
||
|
|
"frac_reward_zero_std": 0.765625,
|
||
|
|
"grad_norm": 0.6154847145080566,
|
||
|
|
"kl": 0.20601044533153376,
|
||
|
|
"learning_rate": 5.63195508137711e-08,
|
||
|
|
"loss": -0.0053,
|
||
|
|
"num_tokens": 22836158.0,
|
||
|
|
"reward": 1.15625,
|
||
|
|
"reward_std": 0.10955046862363815,
|
||
|
|
"rewards/equation_reward_func/mean": 0.15625,
|
||
|
|
"rewards/equation_reward_func/std": 0.36344730854034424,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6115760803222656,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999242424964905,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5363020300865173,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6230578422546387,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0054749916307628155,
|
||
|
|
"step": 356
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.517285105979277e-05,
|
||
|
|
"clip_ratio/high_mean": 6.517285105979277e-05,
|
||
|
|
"clip_ratio/low_mean": 3.2075955661841566e-05,
|
||
|
|
"clip_ratio/low_min": 3.2075955661841566e-05,
|
||
|
|
"clip_ratio/region_mean": 9.724880672163433e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 54.1484375,
|
||
|
|
"completions/mean_terminated_length": 54.1484375,
|
||
|
|
"completions/min_length": 41.0,
|
||
|
|
"completions/min_terminated_length": 41.0,
|
||
|
|
"entropy": 0.10751371085643768,
|
||
|
|
"epoch": 0.09548364153627312,
|
||
|
|
"frac_reward_zero_std": 0.8125,
|
||
|
|
"grad_norm": 0.5332208871841431,
|
||
|
|
"kl": 0.32863597178624737,
|
||
|
|
"learning_rate": 5.4061717663707843e-08,
|
||
|
|
"loss": -0.0087,
|
||
|
|
"num_tokens": 22942002.0,
|
||
|
|
"reward": 1.095703125,
|
||
|
|
"reward_std": 0.07365524023771286,
|
||
|
|
"rewards/equation_reward_func/mean": 0.095703125,
|
||
|
|
"rewards/equation_reward_func/std": 0.2944713830947876,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6140475273132324,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001451969146729,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5362913012504578,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6230778694152832,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005398309323936701,
|
||
|
|
"step": 358
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.128128203873833e-05,
|
||
|
|
"clip_ratio/high_mean": 3.128128203873833e-05,
|
||
|
|
"clip_ratio/low_mean": 6.333636378662454e-05,
|
||
|
|
"clip_ratio/low_min": 6.333636378662454e-05,
|
||
|
|
"clip_ratio/region_mean": 9.461764582536287e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 53.45703125,
|
||
|
|
"completions/mean_terminated_length": 53.45703125,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.10722704066170587,
|
||
|
|
"epoch": 0.09601706970128022,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 0.6317431330680847,
|
||
|
|
"kl": 0.28589126840233803,
|
||
|
|
"learning_rate": 5.1844575530106265e-08,
|
||
|
|
"loss": 0.0084,
|
||
|
|
"num_tokens": 23047372.0,
|
||
|
|
"reward": 1.16796875,
|
||
|
|
"reward_std": 0.15452352166175842,
|
||
|
|
"rewards/equation_reward_func/mean": 0.169921875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3759314715862274,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5585031509399414,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0002102851867676,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.37018513679504395,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.9937520027160645,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005061218049377203,
|
||
|
|
"step": 360
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.611789654526446e-05,
|
||
|
|
"clip_ratio/high_mean": 9.611789654526446e-05,
|
||
|
|
"clip_ratio/low_mean": 6.339250184181664e-05,
|
||
|
|
"clip_ratio/low_min": 6.339250184181664e-05,
|
||
|
|
"clip_ratio/region_mean": 0.0001595103983870811,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 71.0,
|
||
|
|
"completions/max_terminated_length": 71.0,
|
||
|
|
"completions/mean_length": 54.3828125,
|
||
|
|
"completions/mean_terminated_length": 54.3828125,
|
||
|
|
"completions/min_length": 33.0,
|
||
|
|
"completions/min_terminated_length": 33.0,
|
||
|
|
"entropy": 0.11144272796809673,
|
||
|
|
"epoch": 0.09655049786628735,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.6381310224533081,
|
||
|
|
"kl": 0.47150007800923455,
|
||
|
|
"learning_rate": 4.9668584852682134e-08,
|
||
|
|
"loss": 0.0046,
|
||
|
|
"num_tokens": 23153472.0,
|
||
|
|
"reward": 1.146484375,
|
||
|
|
"reward_std": 0.14584828913211823,
|
||
|
|
"rewards/equation_reward_func/mean": 0.146484375,
|
||
|
|
"rewards/equation_reward_func/std": 0.35393697023391724,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4582003355026245,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000150442123413,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5402125120162964,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6157927513122559,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005494322627782822,
|
||
|
|
"step": 362
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00013142186152334843,
|
||
|
|
"clip_ratio/high_mean": 0.00013142186152334843,
|
||
|
|
"clip_ratio/low_mean": 0.0002898183399035285,
|
||
|
|
"clip_ratio/low_min": 0.0002898183399035285,
|
||
|
|
"clip_ratio/region_mean": 0.00042124020142687694,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 86.0,
|
||
|
|
"completions/max_terminated_length": 86.0,
|
||
|
|
"completions/mean_length": 54.052734375,
|
||
|
|
"completions/mean_terminated_length": 54.052734375,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.1067884632696708,
|
||
|
|
"epoch": 0.09708392603129445,
|
||
|
|
"frac_reward_zero_std": 0.625,
|
||
|
|
"grad_norm": 0.7056789398193359,
|
||
|
|
"kl": 0.28939730094538796,
|
||
|
|
"learning_rate": 4.753419752512072e-08,
|
||
|
|
"loss": -0.0051,
|
||
|
|
"num_tokens": 23259243.0,
|
||
|
|
"reward": 1.22265625,
|
||
|
|
"reward_std": 0.16498352587223053,
|
||
|
|
"rewards/equation_reward_func/mean": 0.22265625,
|
||
|
|
"rewards/equation_reward_func/std": 0.41643625497817993,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6293188333511353,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.999923050403595,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6104380488395691,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4935784339904785,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.00535621726885438,
|
||
|
|
"step": 364
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001282519531539745,
|
||
|
|
"clip_ratio/high_mean": 0.0001282519531539745,
|
||
|
|
"clip_ratio/low_mean": 0.0002561479793964989,
|
||
|
|
"clip_ratio/low_min": 0.0002561479793964989,
|
||
|
|
"clip_ratio/region_mean": 0.00038439993255047337,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 72.0,
|
||
|
|
"completions/max_terminated_length": 72.0,
|
||
|
|
"completions/mean_length": 53.712890625,
|
||
|
|
"completions/mean_terminated_length": 53.712890625,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.10322670535080963,
|
||
|
|
"epoch": 0.09761735419630156,
|
||
|
|
"frac_reward_zero_std": 0.578125,
|
||
|
|
"grad_norm": 0.45988044142723083,
|
||
|
|
"kl": 0.2182366508576605,
|
||
|
|
"learning_rate": 4.5441856801230525e-08,
|
||
|
|
"loss": 0.0103,
|
||
|
|
"num_tokens": 23364856.0,
|
||
|
|
"reward": 1.169921875,
|
||
|
|
"reward_std": 0.1906290352344513,
|
||
|
|
"rewards/equation_reward_func/mean": 0.169921875,
|
||
|
|
"rewards/equation_reward_func/std": 0.3759314715862274,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4551732540130615,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998262524604797,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4740542471408844,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7464334964752197,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005158869083970785,
|
||
|
|
"step": 366
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00015984804162548648,
|
||
|
|
"clip_ratio/high_mean": 0.00015984804162548648,
|
||
|
|
"clip_ratio/low_mean": 9.854466124023829e-05,
|
||
|
|
"clip_ratio/low_min": 9.854466124023829e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00025839270286572474,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 75.0,
|
||
|
|
"completions/max_terminated_length": 75.0,
|
||
|
|
"completions/mean_length": 53.77734375,
|
||
|
|
"completions/mean_terminated_length": 53.77734375,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.11221809167828825,
|
||
|
|
"epoch": 0.09815078236130868,
|
||
|
|
"frac_reward_zero_std": 0.625,
|
||
|
|
"grad_norm": 0.7078414559364319,
|
||
|
|
"kl": 0.30065473231176537,
|
||
|
|
"learning_rate": 4.3391997202891825e-08,
|
||
|
|
"loss": -0.0062,
|
||
|
|
"num_tokens": 23470462.0,
|
||
|
|
"reward": 1.1953125,
|
||
|
|
"reward_std": 0.16978386044502258,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1953125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3968288004398346,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5995075702667236,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997912645339966,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47465410828590393,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.745168924331665,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005443467292934656,
|
||
|
|
"step": 368
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.360278348438442e-05,
|
||
|
|
"clip_ratio/high_mean": 6.360278348438442e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 6.360278348438442e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.5,
|
||
|
|
"completions/max_terminated_length": 74.5,
|
||
|
|
"completions/mean_length": 54.5341796875,
|
||
|
|
"completions/mean_terminated_length": 54.5341796875,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.10346044186088774,
|
||
|
|
"epoch": 0.09868421052631579,
|
||
|
|
"frac_reward_zero_std": 0.6796875,
|
||
|
|
"grad_norm": 0.4914635121822357,
|
||
|
|
"kl": 0.5723805690391196,
|
||
|
|
"learning_rate": 4.1385044429817966e-08,
|
||
|
|
"loss": 0.0113,
|
||
|
|
"num_tokens": 23682817.0,
|
||
|
|
"reward": 1.12109375,
|
||
|
|
"reward_std": 0.14466224610805511,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1220703125,
|
||
|
|
"rewards/equation_reward_func/std": 0.32691892981529236,
|
||
|
|
"rewards/format_reward_func/mean": 0.9990234375,
|
||
|
|
"rewards/format_reward_func/std": 0.022097086533904076,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.618697166442871,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997864067554474,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47324445843696594,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.748143196105957,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005407879129052162,
|
||
|
|
"step": 370
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00018945899662665196,
|
||
|
|
"clip_ratio/high_mean": 0.00018945899662665196,
|
||
|
|
"clip_ratio/low_mean": 0.00015853772573690448,
|
||
|
|
"clip_ratio/low_min": 0.00015853772573690448,
|
||
|
|
"clip_ratio/region_mean": 0.00034799672236355644,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 73.0,
|
||
|
|
"completions/max_terminated_length": 73.0,
|
||
|
|
"completions/mean_length": 54.654296875,
|
||
|
|
"completions/mean_terminated_length": 54.654296875,
|
||
|
|
"completions/min_length": 42.0,
|
||
|
|
"completions/min_terminated_length": 42.0,
|
||
|
|
"entropy": 0.11119612202876145,
|
||
|
|
"epoch": 0.0992176386913229,
|
||
|
|
"frac_reward_zero_std": 0.734375,
|
||
|
|
"grad_norm": 0.6408384442329407,
|
||
|
|
"kl": 0.20877801668312815,
|
||
|
|
"learning_rate": 3.942141527114978e-08,
|
||
|
|
"loss": -0.015,
|
||
|
|
"num_tokens": 23789112.0,
|
||
|
|
"reward": 1.15234375,
|
||
|
|
"reward_std": 0.11921608448028564,
|
||
|
|
"rewards/equation_reward_func/mean": 0.15234375,
|
||
|
|
"rewards/equation_reward_func/std": 0.35970520973205566,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.443091869354248,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999127388000488,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5361312031745911,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6233763694763184,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005300004035234451,
|
||
|
|
"step": 372
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.431497038445539e-05,
|
||
|
|
"clip_ratio/high_mean": 6.431497038445539e-05,
|
||
|
|
"clip_ratio/low_mean": 3.135189439894425e-05,
|
||
|
|
"clip_ratio/low_min": 3.135189439894425e-05,
|
||
|
|
"clip_ratio/region_mean": 9.566686478339964e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 82.0,
|
||
|
|
"completions/max_terminated_length": 82.0,
|
||
|
|
"completions/mean_length": 54.119140625,
|
||
|
|
"completions/mean_terminated_length": 54.119140625,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.1074680650813712,
|
||
|
|
"epoch": 0.09975106685633002,
|
||
|
|
"frac_reward_zero_std": 0.59375,
|
||
|
|
"grad_norm": 0.43706050515174866,
|
||
|
|
"kl": 0.20503270087970626,
|
||
|
|
"learning_rate": 3.7501517518899486e-08,
|
||
|
|
"loss": 0.0154,
|
||
|
|
"num_tokens": 23894981.0,
|
||
|
|
"reward": 1.208984375,
|
||
|
|
"reward_std": 0.1902308464050293,
|
||
|
|
"rewards/equation_reward_func/mean": 0.208984375,
|
||
|
|
"rewards/equation_reward_func/std": 0.40698084235191345,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.455344557762146,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.999972939491272,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4449951648712158,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8096919059753418,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0052422829903662205,
|
||
|
|
"step": 374
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012764912551372417,
|
||
|
|
"clip_ratio/high_mean": 0.00012764912551372417,
|
||
|
|
"clip_ratio/low_mean": 6.40415172610018e-05,
|
||
|
|
"clip_ratio/low_min": 6.40415172610018e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00019169064277472594,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 72.0,
|
||
|
|
"completions/max_terminated_length": 72.0,
|
||
|
|
"completions/mean_length": 54.064453125,
|
||
|
|
"completions/mean_terminated_length": 54.064453125,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.10840522083971235,
|
||
|
|
"epoch": 0.10028449502133713,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 0.7811239361763,
|
||
|
|
"kl": 0.3093399351669682,
|
||
|
|
"learning_rate": 3.562574988326342e-08,
|
||
|
|
"loss": -0.0028,
|
||
|
|
"num_tokens": 24000806.0,
|
||
|
|
"reward": 1.142578125,
|
||
|
|
"reward_std": 0.16090275347232819,
|
||
|
|
"rewards/equation_reward_func/mean": 0.142578125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3499840497970581,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6213926076889038,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001095533370972,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.37080225348472595,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.9920864105224609,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005592584144324064,
|
||
|
|
"step": 376
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001946542017524027,
|
||
|
|
"clip_ratio/high_mean": 0.0001946542017524027,
|
||
|
|
"clip_ratio/low_mean": 0.00015795392876801392,
|
||
|
|
"clip_ratio/low_min": 0.00015795392876801392,
|
||
|
|
"clip_ratio/region_mean": 0.00035260813052041665,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 75.0,
|
||
|
|
"completions/max_terminated_length": 75.0,
|
||
|
|
"completions/mean_length": 54.26171875,
|
||
|
|
"completions/mean_terminated_length": 54.26171875,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.11080473537246387,
|
||
|
|
"epoch": 0.10081792318634424,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 0.8036662340164185,
|
||
|
|
"kl": 0.34697790671553874,
|
||
|
|
"learning_rate": 3.379450190982114e-08,
|
||
|
|
"loss": -0.0028,
|
||
|
|
"num_tokens": 24106732.0,
|
||
|
|
"reward": 1.154296875,
|
||
|
|
"reward_std": 0.15801823139190674,
|
||
|
|
"rewards/equation_reward_func/mean": 0.154296875,
|
||
|
|
"rewards/equation_reward_func/std": 0.36158639192581177,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.636888027191162,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999844431877136,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4736279845237732,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7473330497741699,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005764137953519821,
|
||
|
|
"step": 378
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012605664475510517,
|
||
|
|
"clip_ratio/high_mean": 0.00012605664475510517,
|
||
|
|
"clip_ratio/low_mean": 0.00012994798534135852,
|
||
|
|
"clip_ratio/low_min": 0.00012994798534135852,
|
||
|
|
"clip_ratio/region_mean": 0.0002560046300964637,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 71.0,
|
||
|
|
"completions/max_terminated_length": 71.0,
|
||
|
|
"completions/mean_length": 53.802734375,
|
||
|
|
"completions/mean_terminated_length": 53.802734375,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.10849014855921268,
|
||
|
|
"epoch": 0.10135135135135136,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.5985727310180664,
|
||
|
|
"kl": 0.4769698733256923,
|
||
|
|
"learning_rate": 3.2008153898637255e-08,
|
||
|
|
"loss": 0.0001,
|
||
|
|
"num_tokens": 24212399.0,
|
||
|
|
"reward": 1.138671875,
|
||
|
|
"reward_std": 0.12914691865444183,
|
||
|
|
"rewards/equation_reward_func/mean": 0.138671875,
|
||
|
|
"rewards/equation_reward_func/std": 0.34594178199768066,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4897253513336182,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000121831893921,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47605547308921814,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7422208786010742,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005334699060767889,
|
||
|
|
"step": 380
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00013204237782499858,
|
||
|
|
"clip_ratio/high_mean": 0.00013204237782499858,
|
||
|
|
"clip_ratio/low_mean": 0.0002276669579765035,
|
||
|
|
"clip_ratio/low_min": 0.0002276669579765035,
|
||
|
|
"clip_ratio/region_mean": 0.00035970933580150205,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 52.939453125,
|
||
|
|
"completions/mean_terminated_length": 52.939453125,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.10587952389485306,
|
||
|
|
"epoch": 0.10188477951635846,
|
||
|
|
"frac_reward_zero_std": 0.609375,
|
||
|
|
"grad_norm": 0.7023279070854187,
|
||
|
|
"kl": 0.22818097844719887,
|
||
|
|
"learning_rate": 3.026707682528365e-08,
|
||
|
|
"loss": -0.0042,
|
||
|
|
"num_tokens": 24317400.0,
|
||
|
|
"reward": 1.20703125,
|
||
|
|
"reward_std": 0.16314628720283508,
|
||
|
|
"rewards/equation_reward_func/mean": 0.20703125,
|
||
|
|
"rewards/equation_reward_func/std": 0.40557438135147095,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.451087236404419,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000056266784668,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6041624546051025,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.503912091255188,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0053162239491939545,
|
||
|
|
"step": 382
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.1928480085399417e-05,
|
||
|
|
"clip_ratio/high_mean": 3.1928480085399417e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0001602483680471778,
|
||
|
|
"clip_ratio/low_min": 0.0001602483680471778,
|
||
|
|
"clip_ratio/region_mean": 0.0001921768481325772,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 78.0,
|
||
|
|
"completions/max_terminated_length": 78.0,
|
||
|
|
"completions/mean_length": 54.53515625,
|
||
|
|
"completions/mean_terminated_length": 54.53515625,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.10885071423318651,
|
||
|
|
"epoch": 0.10241820768136557,
|
||
|
|
"frac_reward_zero_std": 0.625,
|
||
|
|
"grad_norm": 0.9072684049606323,
|
||
|
|
"kl": 0.3265605628904369,
|
||
|
|
"learning_rate": 2.8571632263797745e-08,
|
||
|
|
"loss": 0.001,
|
||
|
|
"num_tokens": 24423626.0,
|
||
|
|
"reward": 1.1640625,
|
||
|
|
"reward_std": 0.1621546745300293,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1640625,
|
||
|
|
"rewards/equation_reward_func/std": 0.37069445848464966,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4786670207977295,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9996113777160645,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4724961221218109,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7497258186340332,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005308371968567371,
|
||
|
|
"step": 384
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.2679737260979084e-05,
|
||
|
|
"clip_ratio/high_mean": 3.2679737260979084e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00019532104165086316,
|
||
|
|
"clip_ratio/low_min": 0.00019532104165086316,
|
||
|
|
"clip_ratio/region_mean": 0.00022800077891184224,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 79.0,
|
||
|
|
"completions/max_terminated_length": 79.0,
|
||
|
|
"completions/mean_length": 53.27734375,
|
||
|
|
"completions/mean_terminated_length": 53.27734375,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.10969236410326427,
|
||
|
|
"epoch": 0.1029516358463727,
|
||
|
|
"frac_reward_zero_std": 0.6171875,
|
||
|
|
"grad_norm": 0.8136389255523682,
|
||
|
|
"kl": 0.2749815947479672,
|
||
|
|
"learning_rate": 2.6922172311593884e-08,
|
||
|
|
"loss": 0.0013,
|
||
|
|
"num_tokens": 24634038.0,
|
||
|
|
"reward": 1.205078125,
|
||
|
|
"reward_std": 0.16476191580295563,
|
||
|
|
"rewards/equation_reward_func/mean": 0.20703125,
|
||
|
|
"rewards/equation_reward_func/std": 0.4051145315170288,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.03121940791606903,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6781755685806274,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999711513519287,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.598201721906662,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5333266258239746,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005326430080458522,
|
||
|
|
"step": 386
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001562595037588229,
|
||
|
|
"clip_ratio/high_mean": 0.0001562595037588229,
|
||
|
|
"clip_ratio/low_mean": 0.00016435466952518455,
|
||
|
|
"clip_ratio/low_min": 0.00016435466952518455,
|
||
|
|
"clip_ratio/region_mean": 0.0003206141765177664,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 79.0,
|
||
|
|
"completions/max_terminated_length": 79.0,
|
||
|
|
"completions/mean_length": 54.33984375,
|
||
|
|
"completions/mean_terminated_length": 54.33984375,
|
||
|
|
"completions/min_length": 32.0,
|
||
|
|
"completions/min_terminated_length": 32.0,
|
||
|
|
"entropy": 0.10665819586979018,
|
||
|
|
"epoch": 0.1034850640113798,
|
||
|
|
"frac_reward_zero_std": 0.828125,
|
||
|
|
"grad_norm": 0.5512076616287231,
|
||
|
|
"kl": 0.25673069970475304,
|
||
|
|
"learning_rate": 2.5319039516341844e-08,
|
||
|
|
"loss": -0.0046,
|
||
|
|
"num_tokens": 24740132.0,
|
||
|
|
"reward": 1.099609375,
|
||
|
|
"reward_std": 0.07699654996395111,
|
||
|
|
"rewards/equation_reward_func/mean": 0.099609375,
|
||
|
|
"rewards/equation_reward_func/std": 0.29977133870124817,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8591183423995972,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000103235244751,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6897003054618835,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6201024055480957,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005149331409484148,
|
||
|
|
"step": 388
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.239134947665864e-05,
|
||
|
|
"clip_ratio/high_mean": 6.239134947665864e-05,
|
||
|
|
"clip_ratio/low_mean": 9.417718522147172e-05,
|
||
|
|
"clip_ratio/low_min": 9.417718522147172e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00015656853469813036,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 82.0,
|
||
|
|
"completions/max_terminated_length": 82.0,
|
||
|
|
"completions/mean_length": 54.537109375,
|
||
|
|
"completions/mean_terminated_length": 54.537109375,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.10971176334553295,
|
||
|
|
"epoch": 0.10401849217638691,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 0.692693829536438,
|
||
|
|
"kl": 0.2153504646072785,
|
||
|
|
"learning_rate": 2.3762566804829742e-08,
|
||
|
|
"loss": -0.0027,
|
||
|
|
"num_tokens": 24846255.0,
|
||
|
|
"reward": 1.158203125,
|
||
|
|
"reward_std": 0.16714656352996826,
|
||
|
|
"rewards/equation_reward_func/mean": 0.158203125,
|
||
|
|
"rewards/equation_reward_func/std": 0.36528825759887695,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7098453044891357,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0002349615097046,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4768085777759552,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7406401634216309,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0057729813270270824,
|
||
|
|
"step": 390
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.298927681003179e-05,
|
||
|
|
"clip_ratio/high_mean": 9.298927681003179e-05,
|
||
|
|
"clip_ratio/low_mean": 9.58334163038267e-05,
|
||
|
|
"clip_ratio/low_min": 9.58334163038267e-05,
|
||
|
|
"clip_ratio/region_mean": 0.0001888226931138585,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 80.0,
|
||
|
|
"completions/max_terminated_length": 80.0,
|
||
|
|
"completions/mean_length": 53.759765625,
|
||
|
|
"completions/mean_terminated_length": 53.759765625,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.11222462935580148,
|
||
|
|
"epoch": 0.10455192034139403,
|
||
|
|
"frac_reward_zero_std": 0.765625,
|
||
|
|
"grad_norm": 0.6533622145652771,
|
||
|
|
"kl": 0.30716266441676354,
|
||
|
|
"learning_rate": 2.2253077413823458e-08,
|
||
|
|
"loss": 0.0082,
|
||
|
|
"num_tokens": 24951828.0,
|
||
|
|
"reward": 1.12109375,
|
||
|
|
"reward_std": 0.1058690994977951,
|
||
|
|
"rewards/equation_reward_func/mean": 0.12109375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3265552520751953,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 2.0,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998432397842407,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.537998378276825,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6983377933502197,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005584080703556538,
|
||
|
|
"step": 392
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00025420298010835215,
|
||
|
|
"clip_ratio/high_mean": 0.00025420298010835215,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.00025420298010835215,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 80.0,
|
||
|
|
"completions/max_terminated_length": 80.0,
|
||
|
|
"completions/mean_length": 54.27734375,
|
||
|
|
"completions/mean_terminated_length": 54.27734375,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.10937931471400791,
|
||
|
|
"epoch": 0.10508534850640114,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.5494008660316467,
|
||
|
|
"kl": 0.190183505622877,
|
||
|
|
"learning_rate": 2.0790884822939836e-08,
|
||
|
|
"loss": -0.0054,
|
||
|
|
"num_tokens": 25057778.0,
|
||
|
|
"reward": 1.14453125,
|
||
|
|
"reward_std": 0.1324264407157898,
|
||
|
|
"rewards/equation_reward_func/mean": 0.14453125,
|
||
|
|
"rewards/equation_reward_func/std": 0.35197147727012634,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5624616146087646,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000663995742798,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5897343754768372,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.528083086013794,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005392900668084621,
|
||
|
|
"step": 394
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0002244631792159958,
|
||
|
|
"clip_ratio/high_mean": 0.0002244631792159958,
|
||
|
|
"clip_ratio/low_mean": 6.410540663637221e-05,
|
||
|
|
"clip_ratio/low_min": 6.410540663637221e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00028856858585236803,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 75.0,
|
||
|
|
"completions/max_terminated_length": 75.0,
|
||
|
|
"completions/mean_length": 53.62109375,
|
||
|
|
"completions/mean_terminated_length": 53.62109375,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.11050118981964058,
|
||
|
|
"epoch": 0.10561877667140825,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.5563652515411377,
|
||
|
|
"kl": 0.2800249651902252,
|
||
|
|
"learning_rate": 1.9376292689545158e-08,
|
||
|
|
"loss": 0.0042,
|
||
|
|
"num_tokens": 25163296.0,
|
||
|
|
"reward": 1.138671875,
|
||
|
|
"reward_std": 0.13296005129814148,
|
||
|
|
"rewards/equation_reward_func/mean": 0.140625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3479743003845215,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6930088996887207,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000758171081543,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47472190856933594,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7450261116027832,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005275674629956484,
|
||
|
|
"step": 396
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.299023632684516e-05,
|
||
|
|
"clip_ratio/high_mean": 3.299023632684516e-05,
|
||
|
|
"clip_ratio/low_mean": 6.681774134954644e-05,
|
||
|
|
"clip_ratio/low_min": 6.681774134954644e-05,
|
||
|
|
"clip_ratio/region_mean": 9.98079776763916e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 83.0,
|
||
|
|
"completions/max_terminated_length": 83.0,
|
||
|
|
"completions/mean_length": 53.294921875,
|
||
|
|
"completions/mean_terminated_length": 53.294921875,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.10927490248448318,
|
||
|
|
"epoch": 0.10615220483641537,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.6097186207771301,
|
||
|
|
"kl": 0.21666139115889868,
|
||
|
|
"learning_rate": 1.800959478569422e-08,
|
||
|
|
"loss": 0.0028,
|
||
|
|
"num_tokens": 25268447.0,
|
||
|
|
"reward": 1.166015625,
|
||
|
|
"reward_std": 0.12276645004749298,
|
||
|
|
"rewards/equation_reward_func/mean": 0.166015625,
|
||
|
|
"rewards/equation_reward_func/std": 0.3724585771560669,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5868266820907593,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999920129776001,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.41749459505081177,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8734836578369141,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005354772787541151,
|
||
|
|
"step": 398
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.36292723659426e-05,
|
||
|
|
"clip_ratio/high_mean": 3.36292723659426e-05,
|
||
|
|
"clip_ratio/low_mean": 9.74817505468511e-05,
|
||
|
|
"clip_ratio/low_min": 9.74817505468511e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00013111102291279368,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 84.0,
|
||
|
|
"completions/max_terminated_length": 84.0,
|
||
|
|
"completions/mean_length": 54.10546875,
|
||
|
|
"completions/mean_terminated_length": 54.10546875,
|
||
|
|
"completions/min_length": 41.0,
|
||
|
|
"completions/min_terminated_length": 41.0,
|
||
|
|
"entropy": 0.11135919060972002,
|
||
|
|
"epoch": 0.10668563300142248,
|
||
|
|
"frac_reward_zero_std": 0.734375,
|
||
|
|
"grad_norm": 0.5380083918571472,
|
||
|
|
"kl": 0.3635715709792243,
|
||
|
|
"learning_rate": 1.6691074937121407e-08,
|
||
|
|
"loss": -0.0035,
|
||
|
|
"num_tokens": 25374229.0,
|
||
|
|
"reward": 1.12109375,
|
||
|
|
"reward_std": 0.11152878403663635,
|
||
|
|
"rewards/equation_reward_func/mean": 0.123046875,
|
||
|
|
"rewards/equation_reward_func/std": 0.32881227135658264,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.587953805923462,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000246524810791,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6871500015258789,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4624462127685547,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.004950111731886864,
|
||
|
|
"step": 400
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 3.0127742017308872e-05,
|
||
|
|
"clip_ratio/low_min": 3.0127742017308872e-05,
|
||
|
|
"clip_ratio/region_mean": 3.0127742017308872e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 73.5,
|
||
|
|
"completions/max_terminated_length": 73.5,
|
||
|
|
"completions/mean_length": 54.755859375,
|
||
|
|
"completions/mean_terminated_length": 54.755859375,
|
||
|
|
"completions/min_length": 38.5,
|
||
|
|
"completions/min_terminated_length": 38.5,
|
||
|
|
"entropy": 0.10915559923483266,
|
||
|
|
"epoch": 0.10721906116642958,
|
||
|
|
"frac_reward_zero_std": 0.6796875,
|
||
|
|
"grad_norm": 0.5684436559677124,
|
||
|
|
"kl": 0.20094259290231598,
|
||
|
|
"learning_rate": 1.5421006964298377e-08,
|
||
|
|
"loss": -0.0052,
|
||
|
|
"num_tokens": 25586915.0,
|
||
|
|
"reward": 1.142578125,
|
||
|
|
"reward_std": 0.1426895670592785,
|
||
|
|
"rewards/equation_reward_func/mean": 0.142578125,
|
||
|
|
"rewards/equation_reward_func/std": 0.34958136081695557,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4456849694252014,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000089704990387,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.506356343626976,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6825828552246094,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0053960594814270735,
|
||
|
|
"step": 402
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00018773043929185305,
|
||
|
|
"clip_ratio/high_mean": 0.00018773043929185305,
|
||
|
|
"clip_ratio/low_mean": 3.163756142991284e-05,
|
||
|
|
"clip_ratio/low_min": 3.163756142991284e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00021936800072176589,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 78.0,
|
||
|
|
"completions/max_terminated_length": 78.0,
|
||
|
|
"completions/mean_length": 53.34375,
|
||
|
|
"completions/mean_terminated_length": 53.34375,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.10881674703624514,
|
||
|
|
"epoch": 0.1077524893314367,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.5433355569839478,
|
||
|
|
"kl": 0.23106293090515667,
|
||
|
|
"learning_rate": 1.4199654625568575e-08,
|
||
|
|
"loss": 0.0069,
|
||
|
|
"num_tokens": 25692203.0,
|
||
|
|
"reward": 1.126953125,
|
||
|
|
"reward_std": 0.1148107573390007,
|
||
|
|
"rewards/equation_reward_func/mean": 0.126953125,
|
||
|
|
"rewards/equation_reward_func/std": 0.33324605226516724,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6753840446472168,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001516342163086,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47411516308784485,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.746304988861084,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005414518993347883,
|
||
|
|
"step": 404
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.571563426405191e-05,
|
||
|
|
"clip_ratio/high_mean": 6.571563426405191e-05,
|
||
|
|
"clip_ratio/low_mean": 3.233734363069137e-05,
|
||
|
|
"clip_ratio/low_min": 3.233734363069137e-05,
|
||
|
|
"clip_ratio/region_mean": 9.805297789474328e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 71.0,
|
||
|
|
"completions/max_terminated_length": 71.0,
|
||
|
|
"completions/mean_length": 54.330078125,
|
||
|
|
"completions/mean_terminated_length": 54.330078125,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.10553992167115211,
|
||
|
|
"epoch": 0.10828591749644381,
|
||
|
|
"frac_reward_zero_std": 0.578125,
|
||
|
|
"grad_norm": 0.7051840424537659,
|
||
|
|
"kl": 0.25712460403641063,
|
||
|
|
"learning_rate": 1.302727156237224e-08,
|
||
|
|
"loss": -0.0031,
|
||
|
|
"num_tokens": 25798260.0,
|
||
|
|
"reward": 1.212890625,
|
||
|
|
"reward_std": 0.18944953382015228,
|
||
|
|
"rewards/equation_reward_func/mean": 0.21484375,
|
||
|
|
"rewards/equation_reward_func/std": 0.4111155867576599,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6173588037490845,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998912215232849,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5355166792869568,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6245231628417969,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005070769228041172,
|
||
|
|
"step": 406
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001589058018806908,
|
||
|
|
"clip_ratio/high_mean": 0.0001589058018806908,
|
||
|
|
"clip_ratio/low_mean": 6.55909728569289e-05,
|
||
|
|
"clip_ratio/low_min": 6.55909728569289e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00022449677473761968,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 54.080078125,
|
||
|
|
"completions/mean_terminated_length": 54.080078125,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.10741599255965815,
|
||
|
|
"epoch": 0.10881934566145092,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.5971543788909912,
|
||
|
|
"kl": 0.3184280472083224,
|
||
|
|
"learning_rate": 1.1904101246571874e-08,
|
||
|
|
"loss": 0.0085,
|
||
|
|
"num_tokens": 25904117.0,
|
||
|
|
"reward": 1.099609375,
|
||
|
|
"reward_std": 0.1145443245768547,
|
||
|
|
"rewards/equation_reward_func/mean": 0.099609375,
|
||
|
|
"rewards/equation_reward_func/std": 0.29977133870124817,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.46078360080719,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001788139343262,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.41946613788604736,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8687725067138672,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.004916239529848099,
|
||
|
|
"step": 408
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00029215401931044954,
|
||
|
|
"clip_ratio/high_mean": 0.00029215401931044954,
|
||
|
|
"clip_ratio/low_mean": 6.396731219461394e-05,
|
||
|
|
"clip_ratio/low_min": 6.396731219461394e-05,
|
||
|
|
"clip_ratio/region_mean": 0.0003561213315050635,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 72.0,
|
||
|
|
"completions/max_terminated_length": 72.0,
|
||
|
|
"completions/mean_length": 53.44140625,
|
||
|
|
"completions/mean_terminated_length": 53.44140625,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.106752199638221,
|
||
|
|
"epoch": 0.10935277382645804,
|
||
|
|
"frac_reward_zero_std": 0.5625,
|
||
|
|
"grad_norm": 0.6034308075904846,
|
||
|
|
"kl": 0.19704193456305397,
|
||
|
|
"learning_rate": 1.0830376929889612e-08,
|
||
|
|
"loss": -0.002,
|
||
|
|
"num_tokens": 26009479.0,
|
||
|
|
"reward": 1.2109375,
|
||
|
|
"reward_std": 0.19444583356380463,
|
||
|
|
"rewards/equation_reward_func/mean": 0.2109375,
|
||
|
|
"rewards/equation_reward_func/std": 0.4083731174468994,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4476516246795654,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999210834503174,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5376265048980713,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6205911636352539,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005288798827677965,
|
||
|
|
"step": 410
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00013152075957299935,
|
||
|
|
"clip_ratio/high_mean": 0.00013152075957299935,
|
||
|
|
"clip_ratio/low_mean": 0.00012655667119866444,
|
||
|
|
"clip_ratio/low_min": 0.00012655667119866444,
|
||
|
|
"clip_ratio/region_mean": 0.0002580774307716638,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 79.0,
|
||
|
|
"completions/max_terminated_length": 79.0,
|
||
|
|
"completions/mean_length": 53.400390625,
|
||
|
|
"completions/mean_terminated_length": 53.400390625,
|
||
|
|
"completions/min_length": 33.0,
|
||
|
|
"completions/min_terminated_length": 33.0,
|
||
|
|
"entropy": 0.11002827166683143,
|
||
|
|
"epoch": 0.10988620199146515,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.6455569267272949,
|
||
|
|
"kl": 0.20942417076892322,
|
||
|
|
"learning_rate": 9.806321595467598e-09,
|
||
|
|
"loss": -0.0013,
|
||
|
|
"num_tokens": 26114732.0,
|
||
|
|
"reward": 1.24609375,
|
||
|
|
"reward_std": 0.1438756138086319,
|
||
|
|
"rewards/equation_reward_func/mean": 0.24609375,
|
||
|
|
"rewards/equation_reward_func/std": 0.4311550557613373,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5866832733154297,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001810789108276,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6237314343452454,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.47203534841537476,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005659942049533129,
|
||
|
|
"step": 412
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012916017052096626,
|
||
|
|
"clip_ratio/high_mean": 0.00012916017052096626,
|
||
|
|
"clip_ratio/low_mean": 9.72550284738342e-05,
|
||
|
|
"clip_ratio/low_min": 9.72550284738342e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00022641519899480045,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 54.03515625,
|
||
|
|
"completions/mean_terminated_length": 54.03515625,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11343364355464776,
|
||
|
|
"epoch": 0.11041963015647226,
|
||
|
|
"frac_reward_zero_std": 0.71875,
|
||
|
|
"grad_norm": 0.781673789024353,
|
||
|
|
"kl": 0.21153342889414894,
|
||
|
|
"learning_rate": 8.832147911560173e-09,
|
||
|
|
"loss": 0.0025,
|
||
|
|
"num_tokens": 26220518.0,
|
||
|
|
"reward": 1.181640625,
|
||
|
|
"reward_std": 0.12671181559562683,
|
||
|
|
"rewards/equation_reward_func/mean": 0.181640625,
|
||
|
|
"rewards/equation_reward_func/std": 0.38592514395713806,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.8482966423034668,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999876022338867,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.3686046600341797,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.9980306625366211,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.006071026436984539,
|
||
|
|
"step": 414
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.601988570764661e-05,
|
||
|
|
"clip_ratio/high_mean": 6.601988570764661e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00015977946976717148,
|
||
|
|
"clip_ratio/low_min": 0.00015977946976717148,
|
||
|
|
"clip_ratio/region_mean": 0.0002257993554748181,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 81.0,
|
||
|
|
"completions/max_terminated_length": 81.0,
|
||
|
|
"completions/mean_length": 54.458984375,
|
||
|
|
"completions/mean_terminated_length": 54.458984375,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.11500270126594438,
|
||
|
|
"epoch": 0.11095305832147938,
|
||
|
|
"frac_reward_zero_std": 0.625,
|
||
|
|
"grad_norm": 0.5759050846099854,
|
||
|
|
"kl": 0.18094956999023756,
|
||
|
|
"learning_rate": 7.908058187368726e-09,
|
||
|
|
"loss": -0.0042,
|
||
|
|
"num_tokens": 26326585.0,
|
||
|
|
"reward": 1.16796875,
|
||
|
|
"reward_std": 0.1624211072921753,
|
||
|
|
"rewards/equation_reward_func/mean": 0.16796875,
|
||
|
|
"rewards/equation_reward_func/std": 0.374204158782959,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 2.0,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998392462730408,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4201279282569885,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8671960830688477,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005369527265429497,
|
||
|
|
"step": 416
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.738985151362915e-05,
|
||
|
|
"clip_ratio/high_mean": 6.738985151362915e-05,
|
||
|
|
"clip_ratio/low_mean": 3.200204810127616e-05,
|
||
|
|
"clip_ratio/low_min": 3.200204810127616e-05,
|
||
|
|
"clip_ratio/region_mean": 9.939189961490531e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.5,
|
||
|
|
"completions/max_terminated_length": 74.5,
|
||
|
|
"completions/mean_length": 53.6337890625,
|
||
|
|
"completions/mean_terminated_length": 53.6337890625,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11205452390842968,
|
||
|
|
"epoch": 0.11148648648648649,
|
||
|
|
"frac_reward_zero_std": 0.6484375,
|
||
|
|
"grad_norm": 0.6088562607765198,
|
||
|
|
"kl": 0.2512584278980891,
|
||
|
|
"learning_rate": 7.0342443310273665e-09,
|
||
|
|
"loss": -0.003,
|
||
|
|
"num_tokens": 26537490.0,
|
||
|
|
"reward": 1.166015625,
|
||
|
|
"reward_std": 0.1546947881579399,
|
||
|
|
"rewards/equation_reward_func/mean": 0.166015625,
|
||
|
|
"rewards/equation_reward_func/std": 0.37186430394649506,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.714827835559845,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997610151767731,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.3313564211130142,
|
||
|
|
"sampling/sampling_logp_difference/max": 1.1132968664169312,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005667505552992225,
|
||
|
|
"step": 418
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001591860103265693,
|
||
|
|
"clip_ratio/high_mean": 0.0001591860103265693,
|
||
|
|
"clip_ratio/low_mean": 0.00012875603473124406,
|
||
|
|
"clip_ratio/low_min": 0.00012875603473124406,
|
||
|
|
"clip_ratio/region_mean": 0.00028794204505781334,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 79.0,
|
||
|
|
"completions/max_terminated_length": 79.0,
|
||
|
|
"completions/mean_length": 54.89453125,
|
||
|
|
"completions/mean_terminated_length": 54.89453125,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.11335018173687988,
|
||
|
|
"epoch": 0.1120199146514936,
|
||
|
|
"frac_reward_zero_std": 0.65625,
|
||
|
|
"grad_norm": 0.7120157480239868,
|
||
|
|
"kl": 0.21040037108792198,
|
||
|
|
"learning_rate": 6.210887809749099e-09,
|
||
|
|
"loss": 0.0046,
|
||
|
|
"num_tokens": 26643868.0,
|
||
|
|
"reward": 1.154296875,
|
||
|
|
"reward_std": 0.1618293821811676,
|
||
|
|
"rewards/equation_reward_func/mean": 0.15625,
|
||
|
|
"rewards/equation_reward_func/std": 0.36344730854034424,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6395080089569092,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999812841415405,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.609165370464325,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.4956655502319336,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.00603404501453042,
|
||
|
|
"step": 420
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.229974148174127e-05,
|
||
|
|
"clip_ratio/high_mean": 3.229974148174127e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00028592673091528315,
|
||
|
|
"clip_ratio/low_min": 0.00028592673091528315,
|
||
|
|
"clip_ratio/region_mean": 0.00031822647239702445,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 84.0,
|
||
|
|
"completions/max_terminated_length": 84.0,
|
||
|
|
"completions/mean_length": 53.38671875,
|
||
|
|
"completions/mean_terminated_length": 53.38671875,
|
||
|
|
"completions/min_length": 40.0,
|
||
|
|
"completions/min_terminated_length": 40.0,
|
||
|
|
"entropy": 0.11383476294577122,
|
||
|
|
"epoch": 0.11255334281650071,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 0.6631842851638794,
|
||
|
|
"kl": 0.37177798876331913,
|
||
|
|
"learning_rate": 5.4381596121399476e-09,
|
||
|
|
"loss": -0.0003,
|
||
|
|
"num_tokens": 26749146.0,
|
||
|
|
"reward": 1.18359375,
|
||
|
|
"reward_std": 0.16149252653121948,
|
||
|
|
"rewards/equation_reward_func/mean": 0.18359375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3875311613082886,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5404492616653442,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000014305114746,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5083003640174866,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.676682710647583,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005789338611066341,
|
||
|
|
"step": 422
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00022921719937585294,
|
||
|
|
"clip_ratio/high_mean": 0.00022921719937585294,
|
||
|
|
"clip_ratio/low_mean": 3.142282690128518e-05,
|
||
|
|
"clip_ratio/low_min": 3.142282690128518e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00026064002627713815,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 80.0,
|
||
|
|
"completions/max_terminated_length": 80.0,
|
||
|
|
"completions/mean_length": 53.37109375,
|
||
|
|
"completions/mean_terminated_length": 53.37109375,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.11201548162433836,
|
||
|
|
"epoch": 0.11308677098150782,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.7947490811347961,
|
||
|
|
"kl": 0.1786341749959522,
|
||
|
|
"learning_rate": 4.716220212689332e-09,
|
||
|
|
"loss": 0.0087,
|
||
|
|
"num_tokens": 26854416.0,
|
||
|
|
"reward": 1.2109375,
|
||
|
|
"reward_std": 0.14013366401195526,
|
||
|
|
"rewards/equation_reward_func/mean": 0.212890625,
|
||
|
|
"rewards/equation_reward_func/std": 0.409751296043396,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7253990173339844,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001569986343384,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.4751533269882202,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7441177368164062,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.004982165526598692,
|
||
|
|
"step": 424
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0001316617094239013,
|
||
|
|
"clip_ratio/high_mean": 0.0001316617094239013,
|
||
|
|
"clip_ratio/low_mean": 6.346045928593312e-05,
|
||
|
|
"clip_ratio/low_min": 6.346045928593312e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00019512216870983443,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 54.45703125,
|
||
|
|
"completions/mean_terminated_length": 54.45703125,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.10795113237367736,
|
||
|
|
"epoch": 0.11362019914651493,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.6196188926696777,
|
||
|
|
"kl": 0.259122790561782,
|
||
|
|
"learning_rate": 4.045219538443778e-09,
|
||
|
|
"loss": -0.0036,
|
||
|
|
"num_tokens": 26960570.0,
|
||
|
|
"reward": 1.201171875,
|
||
|
|
"reward_std": 0.10888905078172684,
|
||
|
|
"rewards/equation_reward_func/mean": 0.203125,
|
||
|
|
"rewards/equation_reward_func/std": 0.4027182459831238,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.415486216545105,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997314214706421,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5374058485031128,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6210017204284668,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005382740870118141,
|
||
|
|
"step": 426
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012776548586164912,
|
||
|
|
"clip_ratio/high_mean": 0.00012776548586164912,
|
||
|
|
"clip_ratio/low_mean": 9.751586993742321e-05,
|
||
|
|
"clip_ratio/low_min": 9.751586993742321e-05,
|
||
|
|
"clip_ratio/region_mean": 0.0002252813557990723,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 70.0,
|
||
|
|
"completions/max_terminated_length": 70.0,
|
||
|
|
"completions/mean_length": 54.076171875,
|
||
|
|
"completions/mean_terminated_length": 54.076171875,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.10863094880349106,
|
||
|
|
"epoch": 0.11415362731152205,
|
||
|
|
"frac_reward_zero_std": 0.75,
|
||
|
|
"grad_norm": 0.5494850277900696,
|
||
|
|
"kl": 0.19497828578783405,
|
||
|
|
"learning_rate": 3.4252969378714134e-09,
|
||
|
|
"loss": 0.0076,
|
||
|
|
"num_tokens": 27066361.0,
|
||
|
|
"reward": 1.162109375,
|
||
|
|
"reward_std": 0.12052415311336517,
|
||
|
|
"rewards/equation_reward_func/mean": 0.162109375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3689115643501282,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5807867050170898,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001335144042969,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47290024161338806,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.748870849609375,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005263046361505985,
|
||
|
|
"step": 428
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 3.237503309113284e-05,
|
||
|
|
"clip_ratio/high_mean": 3.237503309113284e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0001267518253169126,
|
||
|
|
"clip_ratio/low_min": 0.0001267518253169126,
|
||
|
|
"clip_ratio/region_mean": 0.00015912685840804543,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 73.0,
|
||
|
|
"completions/max_terminated_length": 73.0,
|
||
|
|
"completions/mean_length": 53.65234375,
|
||
|
|
"completions/mean_terminated_length": 53.65234375,
|
||
|
|
"completions/min_length": 37.0,
|
||
|
|
"completions/min_terminated_length": 37.0,
|
||
|
|
"entropy": 0.10955962331758605,
|
||
|
|
"epoch": 0.11468705547652916,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 0.6626308560371399,
|
||
|
|
"kl": 0.3944896838317315,
|
||
|
|
"learning_rate": 2.856581151922943e-09,
|
||
|
|
"loss": -0.0035,
|
||
|
|
"num_tokens": 27171863.0,
|
||
|
|
"reward": 1.1484375,
|
||
|
|
"reward_std": 0.15551269054412842,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1484375,
|
||
|
|
"rewards/equation_reward_func/std": 0.35588082671165466,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.397895097732544,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000016212463379,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.32679349184036255,
|
||
|
|
"sampling/sampling_logp_difference/max": 1.11842679977417,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005313401110470295,
|
||
|
|
"step": 430
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.902074493260847e-05,
|
||
|
|
"clip_ratio/high_mean": 9.902074493260847e-05,
|
||
|
|
"clip_ratio/low_mean": 9.541076724417508e-05,
|
||
|
|
"clip_ratio/low_min": 9.541076724417508e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00019443151217678355,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 78.0,
|
||
|
|
"completions/max_terminated_length": 78.0,
|
||
|
|
"completions/mean_length": 53.7265625,
|
||
|
|
"completions/mean_terminated_length": 53.7265625,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.11242794824971093,
|
||
|
|
"epoch": 0.11522048364153627,
|
||
|
|
"frac_reward_zero_std": 0.734375,
|
||
|
|
"grad_norm": 0.646879255771637,
|
||
|
|
"kl": 0.20790828329821429,
|
||
|
|
"learning_rate": 2.339190287295678e-09,
|
||
|
|
"loss": -0.0056,
|
||
|
|
"num_tokens": 27277387.0,
|
||
|
|
"reward": 1.1484375,
|
||
|
|
"reward_std": 0.12263103574514389,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1484375,
|
||
|
|
"rewards/equation_reward_func/std": 0.35588082671165466,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.63825523853302,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999201893806458,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5359142422676086,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6237812042236328,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005299155134707689,
|
||
|
|
"step": 432
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.528276329239209e-05,
|
||
|
|
"clip_ratio/high_mean": 6.528276329239209e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/region_mean": 6.528276329239209e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 75.0,
|
||
|
|
"completions/max_terminated_length": 75.0,
|
||
|
|
"completions/mean_length": 53.53515625,
|
||
|
|
"completions/mean_terminated_length": 53.53515625,
|
||
|
|
"completions/min_length": 38.0,
|
||
|
|
"completions/min_terminated_length": 38.0,
|
||
|
|
"entropy": 0.1100023119813866,
|
||
|
|
"epoch": 0.11575391180654339,
|
||
|
|
"frac_reward_zero_std": 0.7265625,
|
||
|
|
"grad_norm": 0.3693109452724457,
|
||
|
|
"kl": 0.21212409436702728,
|
||
|
|
"learning_rate": 1.8732317919060715e-09,
|
||
|
|
"loss": -0.0003,
|
||
|
|
"num_tokens": 27488303.0,
|
||
|
|
"reward": 1.1318359375,
|
||
|
|
"reward_std": 0.11800135672092438,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1318359375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3379475623369217,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7036836743354797,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9997938275337219,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5718834400177002,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.5988638401031494,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005505367880687118,
|
||
|
|
"step": 434
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.71159930082245e-05,
|
||
|
|
"clip_ratio/high_mean": 9.71159930082245e-05,
|
||
|
|
"clip_ratio/low_mean": 9.421712214437623e-05,
|
||
|
|
"clip_ratio/low_min": 9.421712214437623e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00019133311515260074,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 77.0,
|
||
|
|
"completions/max_terminated_length": 77.0,
|
||
|
|
"completions/mean_length": 55.1484375,
|
||
|
|
"completions/mean_terminated_length": 55.1484375,
|
||
|
|
"completions/min_length": 34.0,
|
||
|
|
"completions/min_terminated_length": 34.0,
|
||
|
|
"entropy": 0.10632556552688281,
|
||
|
|
"epoch": 0.1162873399715505,
|
||
|
|
"frac_reward_zero_std": 0.640625,
|
||
|
|
"grad_norm": 0.608108401298523,
|
||
|
|
"kl": 0.18701807078388002,
|
||
|
|
"learning_rate": 1.4588024325756788e-09,
|
||
|
|
"loss": 0.0035,
|
||
|
|
"num_tokens": 27594915.0,
|
||
|
|
"reward": 1.138671875,
|
||
|
|
"reward_std": 0.1637316346168518,
|
||
|
|
"rewards/equation_reward_func/mean": 0.138671875,
|
||
|
|
"rewards/equation_reward_func/std": 0.34594178199768066,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4311141967773438,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998852014541626,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5086729526519775,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6759500503540039,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0051868068985641,
|
||
|
|
"step": 436
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.734939602721069e-05,
|
||
|
|
"clip_ratio/high_mean": 9.734939602721069e-05,
|
||
|
|
"clip_ratio/low_mean": 6.28134552648084e-05,
|
||
|
|
"clip_ratio/low_min": 6.28134552648084e-05,
|
||
|
|
"clip_ratio/region_mean": 0.0001601628512920191,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 74.0,
|
||
|
|
"completions/max_terminated_length": 74.0,
|
||
|
|
"completions/mean_length": 54.189453125,
|
||
|
|
"completions/mean_terminated_length": 54.189453125,
|
||
|
|
"completions/min_length": 35.0,
|
||
|
|
"completions/min_terminated_length": 35.0,
|
||
|
|
"entropy": 0.11053359135985374,
|
||
|
|
"epoch": 0.1168207681365576,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.6285626292228699,
|
||
|
|
"kl": 0.24602288214696777,
|
||
|
|
"learning_rate": 1.0959882749354277e-09,
|
||
|
|
"loss": 0.0011,
|
||
|
|
"num_tokens": 27700876.0,
|
||
|
|
"reward": 1.185546875,
|
||
|
|
"reward_std": 0.143285870552063,
|
||
|
|
"rewards/equation_reward_func/mean": 0.185546875,
|
||
|
|
"rewards/equation_reward_func/std": 0.38912075757980347,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.620995283126831,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998245239257812,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.44262897968292236,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8150234222412109,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005426466464996338,
|
||
|
|
"step": 438
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012761998287816014,
|
||
|
|
"clip_ratio/high_mean": 0.00012761998287816014,
|
||
|
|
"clip_ratio/low_mean": 0.000159376420520453,
|
||
|
|
"clip_ratio/low_min": 0.000159376420520453,
|
||
|
|
"clip_ratio/region_mean": 0.00028699640339861315,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 90.0,
|
||
|
|
"completions/max_terminated_length": 90.0,
|
||
|
|
"completions/mean_length": 53.83984375,
|
||
|
|
"completions/mean_terminated_length": 53.83984375,
|
||
|
|
"completions/min_length": 42.0,
|
||
|
|
"completions/min_terminated_length": 42.0,
|
||
|
|
"entropy": 0.10714658308360311,
|
||
|
|
"epoch": 0.11735419630156473,
|
||
|
|
"frac_reward_zero_std": 0.703125,
|
||
|
|
"grad_norm": 0.45735955238342285,
|
||
|
|
"kl": 0.2908731723825137,
|
||
|
|
"learning_rate": 7.848646655519986e-10,
|
||
|
|
"loss": -0.005,
|
||
|
|
"num_tokens": 27806522.0,
|
||
|
|
"reward": 1.203125,
|
||
|
|
"reward_std": 0.1334155797958374,
|
||
|
|
"rewards/equation_reward_func/mean": 0.205078125,
|
||
|
|
"rewards/equation_reward_func/std": 0.4041535556316376,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4523879289627075,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998242259025574,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6908398270606995,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.37320899963378906,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.004832516424357891,
|
||
|
|
"step": 440
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 6.629579002037644e-05,
|
||
|
|
"clip_ratio/high_mean": 6.629579002037644e-05,
|
||
|
|
"clip_ratio/low_mean": 0.00016045565522896746,
|
||
|
|
"clip_ratio/low_min": 0.00016045565522896746,
|
||
|
|
"clip_ratio/region_mean": 0.0002267514452493439,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 83.0,
|
||
|
|
"completions/max_terminated_length": 83.0,
|
||
|
|
"completions/mean_length": 54.224609375,
|
||
|
|
"completions/mean_terminated_length": 54.224609375,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.10644441656768322,
|
||
|
|
"epoch": 0.11788762446657183,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.6941789984703064,
|
||
|
|
"kl": 0.20567356712288326,
|
||
|
|
"learning_rate": 5.254962162804799e-10,
|
||
|
|
"loss": 0.0061,
|
||
|
|
"num_tokens": 27912469.0,
|
||
|
|
"reward": 1.169921875,
|
||
|
|
"reward_std": 0.14243090152740479,
|
||
|
|
"rewards/equation_reward_func/mean": 0.173828125,
|
||
|
|
"rewards/equation_reward_func/std": 0.3793322443962097,
|
||
|
|
"rewards/format_reward_func/mean": 0.99609375,
|
||
|
|
"rewards/format_reward_func/std": 0.06243881583213806,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.7774187326431274,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0001775026321411,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47465693950653076,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7451629638671875,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005035408306866884,
|
||
|
|
"step": 442
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 9.787768552390237e-05,
|
||
|
|
"clip_ratio/high_mean": 9.787768552390237e-05,
|
||
|
|
"clip_ratio/low_mean": 9.694968078595896e-05,
|
||
|
|
"clip_ratio/low_min": 9.694968078595896e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00019482736630986133,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 54.001953125,
|
||
|
|
"completions/mean_terminated_length": 54.001953125,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11240475873152415,
|
||
|
|
"epoch": 0.11842105263157894,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.5599156618118286,
|
||
|
|
"kl": 0.2706406813942724,
|
||
|
|
"learning_rate": 3.1793679084632375e-10,
|
||
|
|
"loss": -0.0021,
|
||
|
|
"num_tokens": 28018262.0,
|
||
|
|
"reward": 1.150390625,
|
||
|
|
"reward_std": 0.14459514617919922,
|
||
|
|
"rewards/equation_reward_func/mean": 0.150390625,
|
||
|
|
"rewards/equation_reward_func/std": 0.35780346393585205,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.6746656894683838,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000989437103271,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.32599928975105286,
|
||
|
|
"sampling/sampling_logp_difference/max": 1.1208600997924805,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005868655163794756,
|
||
|
|
"step": 444
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00012872741596462825,
|
||
|
|
"clip_ratio/high_mean": 0.00012872741596462825,
|
||
|
|
"clip_ratio/low_mean": 3.079576223778228e-05,
|
||
|
|
"clip_ratio/low_min": 3.079576223778228e-05,
|
||
|
|
"clip_ratio/region_mean": 0.00015952317820241055,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 76.0,
|
||
|
|
"completions/max_terminated_length": 76.0,
|
||
|
|
"completions/mean_length": 53.478515625,
|
||
|
|
"completions/mean_terminated_length": 53.478515625,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.11034609129031499,
|
||
|
|
"epoch": 0.11895448079658606,
|
||
|
|
"frac_reward_zero_std": 0.671875,
|
||
|
|
"grad_norm": 0.5812860131263733,
|
||
|
|
"kl": 0.24613611793352497,
|
||
|
|
"learning_rate": 1.6222949365926608e-10,
|
||
|
|
"loss": -0.0054,
|
||
|
|
"num_tokens": 28123555.0,
|
||
|
|
"reward": 1.16796875,
|
||
|
|
"reward_std": 0.14814552664756775,
|
||
|
|
"rewards/equation_reward_func/mean": 0.16796875,
|
||
|
|
"rewards/equation_reward_func/std": 0.374204158782959,
|
||
|
|
"rewards/format_reward_func/mean": 1.0,
|
||
|
|
"rewards/format_reward_func/std": 0.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.4527826309204102,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999824166297913,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.47327739000320435,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.7480735778808594,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0052461326122283936,
|
||
|
|
"step": 446
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.00015984166788661646,
|
||
|
|
"clip_ratio/high_mean": 0.00015984166788661646,
|
||
|
|
"clip_ratio/low_mean": 0.0001612306417276462,
|
||
|
|
"clip_ratio/low_min": 0.0001612306417276462,
|
||
|
|
"clip_ratio/region_mean": 0.00032107230961426266,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 79.0,
|
||
|
|
"completions/max_terminated_length": 79.0,
|
||
|
|
"completions/mean_length": 54.671875,
|
||
|
|
"completions/mean_terminated_length": 54.671875,
|
||
|
|
"completions/min_length": 36.0,
|
||
|
|
"completions/min_terminated_length": 36.0,
|
||
|
|
"entropy": 0.1114695995218224,
|
||
|
|
"epoch": 0.11948790896159317,
|
||
|
|
"frac_reward_zero_std": 0.6875,
|
||
|
|
"grad_norm": 0.6841051578521729,
|
||
|
|
"kl": 0.2143436846219831,
|
||
|
|
"learning_rate": 5.84066608615985e-11,
|
||
|
|
"loss": 0.0066,
|
||
|
|
"num_tokens": 28229851.0,
|
||
|
|
"reward": 1.14453125,
|
||
|
|
"reward_std": 0.14432993531227112,
|
||
|
|
"rewards/equation_reward_func/mean": 0.146484375,
|
||
|
|
"rewards/equation_reward_func/std": 0.35393697023391724,
|
||
|
|
"rewards/format_reward_func/mean": 0.998046875,
|
||
|
|
"rewards/format_reward_func/std": 0.04419417306780815,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.3628417253494263,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9998343586921692,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5373474359512329,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6211104393005371,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005433104000985622,
|
||
|
|
"step": 448
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/low_mean": 2.8965357665179505e-05,
|
||
|
|
"clip_ratio/low_min": 2.8965357665179505e-05,
|
||
|
|
"clip_ratio/region_mean": 2.8965357665179505e-05,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/max_length": 85.5,
|
||
|
|
"completions/max_terminated_length": 85.5,
|
||
|
|
"completions/mean_length": 54.6015625,
|
||
|
|
"completions/mean_terminated_length": 54.6015625,
|
||
|
|
"completions/min_length": 41.0,
|
||
|
|
"completions/min_terminated_length": 41.0,
|
||
|
|
"entropy": 0.11345378682017326,
|
||
|
|
"epoch": 0.12002133712660028,
|
||
|
|
"frac_reward_zero_std": 0.6640625,
|
||
|
|
"grad_norm": 0.5929685235023499,
|
||
|
|
"kl": 0.30157056140402955,
|
||
|
|
"learning_rate": 6.489853613067531e-12,
|
||
|
|
"loss": -0.0012,
|
||
|
|
"num_tokens": 28442211.0,
|
||
|
|
"reward": 1.162109375,
|
||
|
|
"reward_std": 0.14864009618759155,
|
||
|
|
"rewards/equation_reward_func/mean": 0.1630859375,
|
||
|
|
"rewards/equation_reward_func/std": 0.3668737858533859,
|
||
|
|
"rewards/format_reward_func/mean": 0.9990234375,
|
||
|
|
"rewards/format_reward_func/std": 0.022097086533904076,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.5475803017616272,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000160336494446,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.6126877665519714,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.557793140411377,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0051607247442007065,
|
||
|
|
"step": 450
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"epoch": 0.12002133712660028,
|
||
|
|
"step": 450,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.010528181331465021,
|
||
|
|
"train_runtime": 3465.3708,
|
||
|
|
"train_samples_per_second": 37.399,
|
||
|
|
"train_steps_per_second": 0.13
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"logging_steps": 2,
|
||
|
|
"max_steps": 450,
|
||
|
|
"num_input_tokens_seen": 28442211,
|
||
|
|
"num_train_epochs": 1,
|
||
|
|
"save_steps": 25,
|
||
|
|
"stateful_callbacks": {
|
||
|
|
"TrainerControl": {
|
||
|
|
"args": {
|
||
|
|
"should_epoch_stop": false,
|
||
|
|
"should_evaluate": false,
|
||
|
|
"should_log": false,
|
||
|
|
"should_save": true,
|
||
|
|
"should_training_stop": true
|
||
|
|
},
|
||
|
|
"attributes": {}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_batch_size": 16,
|
||
|
|
"trial_name": null,
|
||
|
|
"trial_params": null
|
||
|
|
}
|