695 lines
28 KiB
JSON
695 lines
28 KiB
JSON
|
[
|
||
|
|
{
|
||
|
|
"step": 1,
|
||
|
|
"loss": -0.009,
|
||
|
|
"grad_norm": 12.235088348388672,
|
||
|
|
"learning_rate": 0.0,
|
||
|
|
"num_tokens": 27758.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.7793656587600708,
|
||
|
|
"rewards/reward_total/std": 0.12745577096939087,
|
||
|
|
"rewards/reward_market/mean": 0.25,
|
||
|
|
"rewards/reward_market/std": 0.13440430164337158,
|
||
|
|
"rewards/reward_warehouse/mean": 0.42500001192092896,
|
||
|
|
"rewards/reward_warehouse/std": 0.20160645246505737,
|
||
|
|
"rewards/reward_showroom/mean": 0.10436563193798065,
|
||
|
|
"rewards/reward_showroom/std": 0.069697305560112,
|
||
|
|
"reward": 0.7793656587600708,
|
||
|
|
"reward_std": 0.12745577096939087,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.005470390431582928,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.22214925289154053,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.8007970452308655,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9949374794960022,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.076386570930481,
|
||
|
|
"entropy": 0.028439456821217846,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 19.228480510413647,
|
||
|
|
"epoch": 0.05555555555555555
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 2,
|
||
|
|
"loss": 0.0723,
|
||
|
|
"grad_norm": 60.551937103271484,
|
||
|
|
"learning_rate": 5.000000000000001e-07,
|
||
|
|
"num_tokens": 55324.0,
|
||
|
|
"completions/mean_length": 3.25,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 7.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.25,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 7.0,
|
||
|
|
"rewards/reward_total/mean": 0.719434380531311,
|
||
|
|
"rewards/reward_total/std": 0.1505809724330902,
|
||
|
|
"rewards/reward_market/mean": 0.30000001192092896,
|
||
|
|
"rewards/reward_market/std": 0.17597654461860657,
|
||
|
|
"rewards/reward_warehouse/mean": 0.30000001192092896,
|
||
|
|
"rewards/reward_warehouse/std": 0.17597654461860657,
|
||
|
|
"rewards/reward_showroom/mean": 0.11943437159061432,
|
||
|
|
"rewards/reward_showroom/std": 0.07055392116308212,
|
||
|
|
"reward": 0.719434380531311,
|
||
|
|
"reward_std": 0.1505809724330902,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.01085888221859932,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.28092825412750244,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.8382877111434937,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0260276794433594,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.324359655380249,
|
||
|
|
"entropy": 0.036137547835437545,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 17.92062332853675,
|
||
|
|
"epoch": 0.1111111111111111
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 3,
|
||
|
|
"loss": 0.1153,
|
||
|
|
"grad_norm": 232.934814453125,
|
||
|
|
"learning_rate": 1.0000000000000002e-06,
|
||
|
|
"num_tokens": 83000.0,
|
||
|
|
"completions/mean_length": 3.5,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 7.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.5,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 7.0,
|
||
|
|
"rewards/reward_total/mean": 0.7885687351226807,
|
||
|
|
"rewards/reward_total/std": 0.1279384195804596,
|
||
|
|
"rewards/reward_market/mean": 0.32500001788139343,
|
||
|
|
"rewards/reward_market/std": 0.18837162852287292,
|
||
|
|
"rewards/reward_warehouse/mean": 0.375,
|
||
|
|
"rewards/reward_warehouse/std": 0.20160646736621857,
|
||
|
|
"rewards/reward_showroom/mean": 0.08856874704360962,
|
||
|
|
"rewards/reward_showroom/std": 0.07010025531053543,
|
||
|
|
"reward": 0.7885687351226807,
|
||
|
|
"reward_std": 0.1279384344816208,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.0251829382032156,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.6850378513336182,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.5226751565933228,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0358223915100098,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.9838452339172363,
|
||
|
|
"entropy": 0.03248842835137111,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 17.184778176248074,
|
||
|
|
"epoch": 0.16666666666666666
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 4,
|
||
|
|
"loss": 0.0243,
|
||
|
|
"grad_norm": 13.620709419250488,
|
||
|
|
"learning_rate": 1.5e-06,
|
||
|
|
"num_tokens": 110708.0,
|
||
|
|
"completions/mean_length": 3.125,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 7.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.125,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 7.0,
|
||
|
|
"rewards/reward_total/mean": 0.7762374877929688,
|
||
|
|
"rewards/reward_total/std": 0.13016164302825928,
|
||
|
|
"rewards/reward_market/mean": 0.32499998807907104,
|
||
|
|
"rewards/reward_market/std": 0.18837164342403412,
|
||
|
|
"rewards/reward_warehouse/mean": 0.3500000238418579,
|
||
|
|
"rewards/reward_warehouse/std": 0.1967477649450302,
|
||
|
|
"rewards/reward_showroom/mean": 0.10123749077320099,
|
||
|
|
"rewards/reward_showroom/std": 0.06825561076402664,
|
||
|
|
"reward": 0.7762374877929688,
|
||
|
|
"reward_std": 0.13016162812709808,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.007013080175966024,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.2646750509738922,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.7674550414085388,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9837819337844849,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.0260045528411865,
|
||
|
|
"entropy": 0.03743034108288157,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 17.90316915512085,
|
||
|
|
"epoch": 0.2222222222222222
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 5,
|
||
|
|
"loss": 0.0004,
|
||
|
|
"grad_norm": 2.0591225624084473,
|
||
|
|
"learning_rate": 2.0000000000000003e-06,
|
||
|
|
"num_tokens": 138452.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.7784374952316284,
|
||
|
|
"rewards/reward_total/std": 0.12858590483665466,
|
||
|
|
"rewards/reward_market/mean": 0.30000001192092896,
|
||
|
|
"rewards/reward_market/std": 0.17597654461860657,
|
||
|
|
"rewards/reward_warehouse/mean": 0.375,
|
||
|
|
"rewards/reward_warehouse/std": 0.20160646736621857,
|
||
|
|
"rewards/reward_showroom/mean": 0.10343749821186066,
|
||
|
|
"rewards/reward_showroom/std": 0.06380020827054977,
|
||
|
|
"reward": 0.7784374952316284,
|
||
|
|
"reward_std": 0.12858593463897705,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.001465568202547729,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.05673474818468094,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.9749767780303955,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0017430782318115,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.058376669883728,
|
||
|
|
"entropy": 0.008188390799773515,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 17.8210555203259,
|
||
|
|
"epoch": 0.2777777777777778
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 6,
|
||
|
|
"loss": 0.0915,
|
||
|
|
"grad_norm": 31.611696243286133,
|
||
|
|
"learning_rate": 2.5e-06,
|
||
|
|
"num_tokens": 166252.0,
|
||
|
|
"completions/mean_length": 3.125,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 7.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.125,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 7.0,
|
||
|
|
"rewards/reward_total/mean": 0.7915937900543213,
|
||
|
|
"rewards/reward_total/std": 0.12960509955883026,
|
||
|
|
"rewards/reward_market/mean": 0.375,
|
||
|
|
"rewards/reward_market/std": 0.20160646736621857,
|
||
|
|
"rewards/reward_warehouse/mean": 0.32500001788139343,
|
||
|
|
"rewards/reward_warehouse/std": 0.18837162852287292,
|
||
|
|
"rewards/reward_showroom/mean": 0.09159374982118607,
|
||
|
|
"rewards/reward_showroom/std": 0.0639258474111557,
|
||
|
|
"reward": 0.7915937900543213,
|
||
|
|
"reward_std": 0.12960509955883026,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 0.008641102351248264,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.8236088752746582,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.9935171008110046,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0399717092514038,
|
||
|
|
"sampling/importance_sampling_ratio/max": 2.278707504272461,
|
||
|
|
"entropy": 0.007084679029730978,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 18.056264080107212,
|
||
|
|
"epoch": 0.3333333333333333
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 7,
|
||
|
|
"loss": -0.0,
|
||
|
|
"grad_norm": 0.281630277633667,
|
||
|
|
"learning_rate": 3e-06,
|
||
|
|
"num_tokens": 193950.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.7600874900817871,
|
||
|
|
"rewards/reward_total/std": 0.13816162943840027,
|
||
|
|
"rewards/reward_market/mean": 0.32500001788139343,
|
||
|
|
"rewards/reward_market/std": 0.18837162852287292,
|
||
|
|
"rewards/reward_warehouse/mean": 0.32500001788139343,
|
||
|
|
"rewards/reward_warehouse/std": 0.18837162852287292,
|
||
|
|
"rewards/reward_showroom/mean": 0.11008749902248383,
|
||
|
|
"rewards/reward_showroom/std": 0.07028691470623016,
|
||
|
|
"reward": 0.7600874900817871,
|
||
|
|
"reward_std": 0.13816164433956146,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 3.554227441782132e-05,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.0027569520752876997,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.997247576713562,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999052286148071,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.0000724792480469,
|
||
|
|
"entropy": 0.0005298306713825696,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 17.43799263238907,
|
||
|
|
"epoch": 0.3888888888888889
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 8,
|
||
|
|
"loss": 0.0,
|
||
|
|
"grad_norm": 0.00018881642608903348,
|
||
|
|
"learning_rate": 3.5e-06,
|
||
|
|
"num_tokens": 221677.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.7622687816619873,
|
||
|
|
"rewards/reward_total/std": 0.1372590959072113,
|
||
|
|
"rewards/reward_market/mean": 0.4000000059604645,
|
||
|
|
"rewards/reward_market/std": 0.20320022106170654,
|
||
|
|
"rewards/reward_warehouse/mean": 0.25,
|
||
|
|
"rewards/reward_warehouse/std": 0.13440430164337158,
|
||
|
|
"rewards/reward_showroom/mean": 0.11226874589920044,
|
||
|
|
"rewards/reward_showroom/std": 0.07360353320837021,
|
||
|
|
"reward": 0.7622687816619873,
|
||
|
|
"reward_std": 0.1372590959072113,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 2.545601773817907e-07,
|
||
|
|
"sampling/sampling_logp_difference/max": 1.5496943888138048e-06,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.9999986886978149,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.000000238418579,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.0000016689300537,
|
||
|
|
"entropy": 4.1391018498870835e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 17.406394600868225,
|
||
|
|
"epoch": 0.4444444444444444
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 9,
|
||
|
|
"loss": 0.0,
|
||
|
|
"grad_norm": 0.00015632262511644512,
|
||
|
|
"learning_rate": 4.000000000000001e-06,
|
||
|
|
"num_tokens": 249549.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.775946855545044,
|
||
|
|
"rewards/reward_total/std": 0.137176051735878,
|
||
|
|
"rewards/reward_market/mean": 0.375,
|
||
|
|
"rewards/reward_market/std": 0.20160646736621857,
|
||
|
|
"rewards/reward_warehouse/mean": 0.30000001192092896,
|
||
|
|
"rewards/reward_warehouse/std": 0.17597654461860657,
|
||
|
|
"rewards/reward_showroom/mean": 0.10094687342643738,
|
||
|
|
"rewards/reward_showroom/std": 0.058497413992881775,
|
||
|
|
"reward": 0.775946855545044,
|
||
|
|
"reward_std": 0.1371760368347168,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 2.918131087881193e-07,
|
||
|
|
"sampling/sampling_logp_difference/max": 3.099441755693988e-06,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.9999978542327881,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.000001311302185,
|
||
|
|
"entropy": 4.173239403826301e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 18.514019537717104,
|
||
|
|
"epoch": 0.5
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 10,
|
||
|
|
"loss": 0.0,
|
||
|
|
"grad_norm": 9.833038348006085e-05,
|
||
|
|
"learning_rate": 4.5e-06,
|
||
|
|
"num_tokens": 277377.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.7337374687194824,
|
||
|
|
"rewards/reward_total/std": 0.15055809915065765,
|
||
|
|
"rewards/reward_market/mean": 0.2750000059604645,
|
||
|
|
"rewards/reward_market/std": 0.1586231142282486,
|
||
|
|
"rewards/reward_warehouse/mean": 0.3500000238418579,
|
||
|
|
"rewards/reward_warehouse/std": 0.1967477649450302,
|
||
|
|
"rewards/reward_showroom/mean": 0.10873749852180481,
|
||
|
|
"rewards/reward_showroom/std": 0.06778524816036224,
|
||
|
|
"reward": 0.7337374687194824,
|
||
|
|
"reward_std": 0.15055811405181885,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 2.719489771152439e-07,
|
||
|
|
"sampling/sampling_logp_difference/max": 1.9073031580774114e-06,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.9999985694885254,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999998807907104,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.0000019073486328,
|
||
|
|
"entropy": 4.224909940830912e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 18.26371632888913,
|
||
|
|
"epoch": 0.5555555555555556
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 11,
|
||
|
|
"loss": -0.0,
|
||
|
|
"grad_norm": 0.00015879125567153096,
|
||
|
|
"learning_rate": 5e-06,
|
||
|
|
"num_tokens": 305332.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.7567968368530273,
|
||
|
|
"rewards/reward_total/std": 0.13728150725364685,
|
||
|
|
"rewards/reward_market/mean": 0.375,
|
||
|
|
"rewards/reward_market/std": 0.20160646736621857,
|
||
|
|
"rewards/reward_warehouse/mean": 0.2750000059604645,
|
||
|
|
"rewards/reward_warehouse/std": 0.1586231142282486,
|
||
|
|
"rewards/reward_showroom/mean": 0.10679687559604645,
|
||
|
|
"rewards/reward_showroom/std": 0.07404065877199173,
|
||
|
|
"reward": 0.7567968368530273,
|
||
|
|
"reward_std": 0.13728150725364685,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 3.067227396513772e-07,
|
||
|
|
"sampling/sampling_logp_difference/max": 2.6226434783893637e-06,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.9999973773956299,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999994039535522,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.0000019073486328,
|
||
|
|
"entropy": 4.7876037001515215e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 19.4472255371511,
|
||
|
|
"epoch": 0.6111111111111112
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 12,
|
||
|
|
"loss": -0.0,
|
||
|
|
"grad_norm": 0.00039661259506829083,
|
||
|
|
"learning_rate": 4.3750000000000005e-06,
|
||
|
|
"num_tokens": 333165.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.7081500291824341,
|
||
|
|
"rewards/reward_total/std": 0.1421954482793808,
|
||
|
|
"rewards/reward_market/mean": 0.30000001192092896,
|
||
|
|
"rewards/reward_market/std": 0.17597654461860657,
|
||
|
|
"rewards/reward_warehouse/mean": 0.2750000059604645,
|
||
|
|
"rewards/reward_warehouse/std": 0.1586231142282486,
|
||
|
|
"rewards/reward_showroom/mean": 0.13315001130104065,
|
||
|
|
"rewards/reward_showroom/std": 0.07370516657829285,
|
||
|
|
"reward": 0.7081500291824341,
|
||
|
|
"reward_std": 0.142195463180542,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 4.768499479723687e-07,
|
||
|
|
"sampling/sampling_logp_difference/max": 5.602954843197949e-06,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.9999943971633911,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999988675117493,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.0000016689300537,
|
||
|
|
"entropy": 5.9777358274004655e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 18.45015063509345,
|
||
|
|
"epoch": 0.6666666666666666
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 13,
|
||
|
|
"loss": 0.0,
|
||
|
|
"grad_norm": 0.0005282312049530447,
|
||
|
|
"learning_rate": 3.7500000000000005e-06,
|
||
|
|
"num_tokens": 361006.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.7762781381607056,
|
||
|
|
"rewards/reward_total/std": 0.13823458552360535,
|
||
|
|
"rewards/reward_market/mean": 0.32500001788139343,
|
||
|
|
"rewards/reward_market/std": 0.18837162852287292,
|
||
|
|
"rewards/reward_warehouse/mean": 0.3500000238418579,
|
||
|
|
"rewards/reward_warehouse/std": 0.1967477649450302,
|
||
|
|
"rewards/reward_showroom/mean": 0.10127812623977661,
|
||
|
|
"rewards/reward_showroom/std": 0.06158862262964249,
|
||
|
|
"reward": 0.7762781381607056,
|
||
|
|
"reward_std": 0.13823460042476654,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 6.830008487668238e-07,
|
||
|
|
"sampling/sampling_logp_difference/max": 5.60290391149465e-06,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.9999943971633911,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999983310699463,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.0000022649765015,
|
||
|
|
"entropy": 7.182803437899565e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 17.955969959497452,
|
||
|
|
"epoch": 0.7222222222222222
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 14,
|
||
|
|
"loss": -0.0,
|
||
|
|
"grad_norm": 0.000535853614564985,
|
||
|
|
"learning_rate": 3.125e-06,
|
||
|
|
"num_tokens": 388862.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.7628874778747559,
|
||
|
|
"rewards/reward_total/std": 0.1295449286699295,
|
||
|
|
"rewards/reward_market/mean": 0.30000001192092896,
|
||
|
|
"rewards/reward_market/std": 0.17597654461860657,
|
||
|
|
"rewards/reward_warehouse/mean": 0.3499999940395355,
|
||
|
|
"rewards/reward_warehouse/std": 0.19674775004386902,
|
||
|
|
"rewards/reward_showroom/mean": 0.11288750171661377,
|
||
|
|
"rewards/reward_showroom/std": 0.073255755007267,
|
||
|
|
"reward": 0.7628874778747559,
|
||
|
|
"reward_std": 0.1295449435710907,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 1.1113726259281975e-06,
|
||
|
|
"sampling/sampling_logp_difference/max": 2.0979605324100703e-05,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.9999922513961792,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 1.0000004768371582,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.0000211000442505,
|
||
|
|
"entropy": 9.972968496185786e-05,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 18.99697282537818,
|
||
|
|
"epoch": 0.7777777777777778
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 15,
|
||
|
|
"loss": 0.0,
|
||
|
|
"grad_norm": 0.005312301218509674,
|
||
|
|
"learning_rate": 2.5e-06,
|
||
|
|
"num_tokens": 416636.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.7899656295776367,
|
||
|
|
"rewards/reward_total/std": 0.1296130269765854,
|
||
|
|
"rewards/reward_market/mean": 0.3499999940395355,
|
||
|
|
"rewards/reward_market/std": 0.19674775004386902,
|
||
|
|
"rewards/reward_warehouse/mean": 0.3500000238418579,
|
||
|
|
"rewards/reward_warehouse/std": 0.1967477649450302,
|
||
|
|
"rewards/reward_showroom/mean": 0.08996562659740448,
|
||
|
|
"rewards/reward_showroom/std": 0.05463240668177605,
|
||
|
|
"reward": 0.7899656295776367,
|
||
|
|
"reward_std": 0.12961304187774658,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 4.004845322924666e-06,
|
||
|
|
"sampling/sampling_logp_difference/max": 4.589592936099507e-05,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.9999337792396545,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999885559082031,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.0000029802322388,
|
||
|
|
"entropy": 0.0001807671503684105,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 19.184648096561432,
|
||
|
|
"epoch": 0.8333333333333334
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 16,
|
||
|
|
"loss": -0.0,
|
||
|
|
"grad_norm": 0.005995223298668861,
|
||
|
|
"learning_rate": 1.8750000000000003e-06,
|
||
|
|
"num_tokens": 444544.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.789996862411499,
|
||
|
|
"rewards/reward_total/std": 0.12048782408237457,
|
||
|
|
"rewards/reward_market/mean": 0.375,
|
||
|
|
"rewards/reward_market/std": 0.20160646736621857,
|
||
|
|
"rewards/reward_warehouse/mean": 0.32039374113082886,
|
||
|
|
"rewards/reward_warehouse/std": 0.18316024541854858,
|
||
|
|
"rewards/reward_showroom/mean": 0.09460312128067017,
|
||
|
|
"rewards/reward_showroom/std": 0.062435995787382126,
|
||
|
|
"reward": 0.789996862411499,
|
||
|
|
"reward_std": 0.12048781663179398,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 4.866625204158481e-06,
|
||
|
|
"sampling/sampling_logp_difference/max": 2.9087463190080598e-05,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.999959409236908,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999855160713196,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.0000004768371582,
|
||
|
|
"entropy": 0.00019320984370096994,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 19.476148523390293,
|
||
|
|
"epoch": 0.8888888888888888
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 17,
|
||
|
|
"loss": 0.0,
|
||
|
|
"grad_norm": 0.011378168128430843,
|
||
|
|
"learning_rate": 1.25e-06,
|
||
|
|
"num_tokens": 472481.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.8168656826019287,
|
||
|
|
"rewards/reward_total/std": 0.09262391924858093,
|
||
|
|
"rewards/reward_market/mean": 0.375,
|
||
|
|
"rewards/reward_market/std": 0.20160646736621857,
|
||
|
|
"rewards/reward_warehouse/mean": 0.3498094081878662,
|
||
|
|
"rewards/reward_warehouse/std": 0.19690066576004028,
|
||
|
|
"rewards/reward_showroom/mean": 0.09205625206232071,
|
||
|
|
"rewards/reward_showroom/std": 0.07046373933553696,
|
||
|
|
"reward": 0.8168656826019287,
|
||
|
|
"reward_std": 0.09262390434741974,
|
||
|
|
"frac_reward_zero_std": 0.0,
|
||
|
|
"sampling/sampling_logp_difference/mean": 5.950785180175444e-06,
|
||
|
|
"sampling/sampling_logp_difference/max": 4.7328881919384e-05,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.9999284744262695,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999822974205017,
|
||
|
|
"sampling/importance_sampling_ratio/max": 1.000001072883606,
|
||
|
|
"entropy": 0.00021975090658088448,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 19.344543006271124,
|
||
|
|
"epoch": 0.9444444444444444
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 18,
|
||
|
|
"loss": 0.0,
|
||
|
|
"grad_norm": 0.018683306872844696,
|
||
|
|
"learning_rate": 6.25e-07,
|
||
|
|
"num_tokens": 500266.0,
|
||
|
|
"completions/mean_length": 3.0,
|
||
|
|
"completions/min_length": 3.0,
|
||
|
|
"completions/max_length": 3.0,
|
||
|
|
"completions/clipped_ratio": 0.0,
|
||
|
|
"completions/mean_terminated_length": 3.0,
|
||
|
|
"completions/min_terminated_length": 3.0,
|
||
|
|
"completions/max_terminated_length": 3.0,
|
||
|
|
"rewards/reward_total/mean": 0.7750625014305115,
|
||
|
|
"rewards/reward_total/std": 0.14084643125534058,
|
||
|
|
"rewards/reward_market/mean": 0.3500000238418579,
|
||
|
|
"rewards/reward_market/std": 0.1967477649450302,
|
||
|
|
"rewards/reward_warehouse/mean": 0.32499998807907104,
|
||
|
|
"rewards/reward_warehouse/std": 0.18837164342403412,
|
||
|
|
"rewards/reward_showroom/mean": 0.10006250441074371,
|
||
|
|
"rewards/reward_showroom/std": 0.05640558898448944,
|
||
|
|
"reward": 0.7750625014305115,
|
||
|
|
"reward_std": 0.14084644615650177,
|
||
|
|
"frac_reward_zero_std": 0.0625,
|
||
|
|
"sampling/sampling_logp_difference/mean": 1.1303089195280336e-05,
|
||
|
|
"sampling/sampling_logp_difference/max": 0.00027322862297296524,
|
||
|
|
"sampling/importance_sampling_ratio/min": 0.999584436416626,
|
||
|
|
"sampling/importance_sampling_ratio/mean": 0.9999662637710571,
|
||
|
|
"sampling/importance_sampling_ratio/max": 0.9999997615814209,
|
||
|
|
"entropy": 0.000320568448614722,
|
||
|
|
"clip_ratio/low_mean": 0.0,
|
||
|
|
"clip_ratio/low_min": 0.0,
|
||
|
|
"clip_ratio/high_mean": 0.0,
|
||
|
|
"clip_ratio/high_max": 0.0,
|
||
|
|
"clip_ratio/region_mean": 0.0,
|
||
|
|
"step_time": 17.66909484937787,
|
||
|
|
"epoch": 1.0
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"step": 18,
|
||
|
|
"train_runtime": 406.352,
|
||
|
|
"train_samples_per_second": 0.738,
|
||
|
|
"train_steps_per_second": 0.044,
|
||
|
|
"total_flos": 0.0,
|
||
|
|
"train_loss": 0.016376476217475202,
|
||
|
|
"epoch": 1.0
|
||
|
|
}
|
||
|
|
]
|