hard007ik/shopmanager-grpo-qwen3
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

初始化项目，由ModelHub XC社区提供模型

Browse Source 
Model: hard007ik/shopmanager-grpo-qwen3
Source: Original Platform
This commit is contained in:
ModelHub XC
2026-05-30 03:33:19 +08:00
commit 2d648244dd
27 changed files with 153839 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
training_artifacts/loss_curve.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 40 KiB

20
training_artifacts/metrics.csv Normal file
View File

@@ -0,0 +1,20 @@
step,loss,grad_norm,learning_rate,num_tokens,completions/mean_length,completions/min_length,completions/max_length,completions/clipped_ratio,completions/mean_terminated_length,completions/min_terminated_length,completions/max_terminated_length,rewards/reward_total/mean,rewards/reward_total/std,rewards/reward_market/mean,rewards/reward_market/std,rewards/reward_warehouse/mean,rewards/reward_warehouse/std,rewards/reward_showroom/mean,rewards/reward_showroom/std,reward,reward_std,frac_reward_zero_std,sampling/sampling_logp_difference/mean,sampling/sampling_logp_difference/max,sampling/importance_sampling_ratio/min,sampling/importance_sampling_ratio/mean,sampling/importance_sampling_ratio/max,entropy,clip_ratio/low_mean,clip_ratio/low_min,clip_ratio/high_mean,clip_ratio/high_max,clip_ratio/region_mean,step_time,epoch,train_runtime,train_samples_per_second,train_steps_per_second,total_flos,train_loss
1,-0.009,12.235088348388672,0.0,27758.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.7793656587600708,0.12745577096939087,0.25,0.13440430164337158,0.42500001192092896,0.20160645246505737,0.10436563193798065,0.069697305560112,0.7793656587600708,0.12745577096939087,0.0,0.005470390431582928,0.22214925289154053,0.8007970452308655,0.9949374794960022,1.076386570930481,0.028439456821217846,0.0,0.0,0.0,0.0,0.0,19.228480510413647,0.05555555555555555,,,,,
2,0.0723,60.551937103271484,5.000000000000001e-07,55324.0,3.25,3.0,7.0,0.0,3.25,3.0,7.0,0.719434380531311,0.1505809724330902,0.30000001192092896,0.17597654461860657,0.30000001192092896,0.17597654461860657,0.11943437159061432,0.07055392116308212,0.719434380531311,0.1505809724330902,0.0,0.01085888221859932,0.28092825412750244,0.8382877111434937,1.0260276794433594,1.324359655380249,0.036137547835437545,0.0,0.0,0.0,0.0,0.0,17.92062332853675,0.1111111111111111,,,,,
3,0.1153,232.934814453125,1.0000000000000002e-06,83000.0,3.5,3.0,7.0,0.0,3.5,3.0,7.0,0.7885687351226807,0.1279384195804596,0.32500001788139343,0.18837162852287292,0.375,0.20160646736621857,0.08856874704360962,0.07010025531053543,0.7885687351226807,0.1279384344816208,0.0,0.0251829382032156,0.6850378513336182,0.5226751565933228,1.0358223915100098,1.9838452339172363,0.03248842835137111,0.0,0.0,0.0,0.0,0.0,17.184778176248074,0.16666666666666666,,,,,
4,0.0243,13.620709419250488,1.5e-06,110708.0,3.125,3.0,7.0,0.0,3.125,3.0,7.0,0.7762374877929688,0.13016164302825928,0.32499998807907104,0.18837164342403412,0.3500000238418579,0.1967477649450302,0.10123749077320099,0.06825561076402664,0.7762374877929688,0.13016162812709808,0.0,0.007013080175966024,0.2646750509738922,0.7674550414085388,0.9837819337844849,1.0260045528411865,0.03743034108288157,0.0,0.0,0.0,0.0,0.0,17.90316915512085,0.2222222222222222,,,,,
5,0.0004,2.0591225624084473,2.0000000000000003e-06,138452.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.7784374952316284,0.12858590483665466,0.30000001192092896,0.17597654461860657,0.375,0.20160646736621857,0.10343749821186066,0.06380020827054977,0.7784374952316284,0.12858593463897705,0.0,0.001465568202547729,0.05673474818468094,0.9749767780303955,1.0017430782318115,1.058376669883728,0.008188390799773515,0.0,0.0,0.0,0.0,0.0,17.8210555203259,0.2777777777777778,,,,,
6,0.0915,31.611696243286133,2.5e-06,166252.0,3.125,3.0,7.0,0.0,3.125,3.0,7.0,0.7915937900543213,0.12960509955883026,0.375,0.20160646736621857,0.32500001788139343,0.18837162852287292,0.09159374982118607,0.0639258474111557,0.7915937900543213,0.12960509955883026,0.0,0.008641102351248264,0.8236088752746582,0.9935171008110046,1.0399717092514038,2.278707504272461,0.007084679029730978,0.0,0.0,0.0,0.0,0.0,18.056264080107212,0.3333333333333333,,,,,
7,-0.0,0.281630277633667,3e-06,193950.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.7600874900817871,0.13816162943840027,0.32500001788139343,0.18837162852287292,0.32500001788139343,0.18837162852287292,0.11008749902248383,0.07028691470623016,0.7600874900817871,0.13816164433956146,0.0,3.554227441782132e-05,0.0027569520752876997,0.997247576713562,0.9999052286148071,1.0000724792480469,0.0005298306713825696,0.0,0.0,0.0,0.0,0.0,17.43799263238907,0.3888888888888889,,,,,
8,0.0,0.00018881642608903348,3.5e-06,221677.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.7622687816619873,0.1372590959072113,0.4000000059604645,0.20320022106170654,0.25,0.13440430164337158,0.11226874589920044,0.07360353320837021,0.7622687816619873,0.1372590959072113,0.0,2.545601773817907e-07,1.5496943888138048e-06,0.9999986886978149,1.000000238418579,1.0000016689300537,4.1391018498870835e-05,0.0,0.0,0.0,0.0,0.0,17.406394600868225,0.4444444444444444,,,,,
9,0.0,0.00015632262511644512,4.000000000000001e-06,249549.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.775946855545044,0.137176051735878,0.375,0.20160646736621857,0.30000001192092896,0.17597654461860657,0.10094687342643738,0.058497413992881775,0.775946855545044,0.1371760368347168,0.0,2.918131087881193e-07,3.099441755693988e-06,0.9999978542327881,1.0,1.000001311302185,4.173239403826301e-05,0.0,0.0,0.0,0.0,0.0,18.514019537717104,0.5,,,,,
10,0.0,9.833038348006085e-05,4.5e-06,277377.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.7337374687194824,0.15055809915065765,0.2750000059604645,0.1586231142282486,0.3500000238418579,0.1967477649450302,0.10873749852180481,0.06778524816036224,0.7337374687194824,0.15055811405181885,0.0,2.719489771152439e-07,1.9073031580774114e-06,0.9999985694885254,0.9999998807907104,1.0000019073486328,4.224909940830912e-05,0.0,0.0,0.0,0.0,0.0,18.26371632888913,0.5555555555555556,,,,,
11,-0.0,0.00015879125567153096,5e-06,305332.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.7567968368530273,0.13728150725364685,0.375,0.20160646736621857,0.2750000059604645,0.1586231142282486,0.10679687559604645,0.07404065877199173,0.7567968368530273,0.13728150725364685,0.0,3.067227396513772e-07,2.6226434783893637e-06,0.9999973773956299,0.9999994039535522,1.0000019073486328,4.7876037001515215e-05,0.0,0.0,0.0,0.0,0.0,19.4472255371511,0.6111111111111112,,,,,
12,-0.0,0.00039661259506829083,4.3750000000000005e-06,333165.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.7081500291824341,0.1421954482793808,0.30000001192092896,0.17597654461860657,0.2750000059604645,0.1586231142282486,0.13315001130104065,0.07370516657829285,0.7081500291824341,0.142195463180542,0.0,4.768499479723687e-07,5.602954843197949e-06,0.9999943971633911,0.9999988675117493,1.0000016689300537,5.9777358274004655e-05,0.0,0.0,0.0,0.0,0.0,18.45015063509345,0.6666666666666666,,,,,
13,0.0,0.0005282312049530447,3.7500000000000005e-06,361006.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.7762781381607056,0.13823458552360535,0.32500001788139343,0.18837162852287292,0.3500000238418579,0.1967477649450302,0.10127812623977661,0.06158862262964249,0.7762781381607056,0.13823460042476654,0.0,6.830008487668238e-07,5.60290391149465e-06,0.9999943971633911,0.9999983310699463,1.0000022649765015,7.182803437899565e-05,0.0,0.0,0.0,0.0,0.0,17.955969959497452,0.7222222222222222,,,,,
14,-0.0,0.000535853614564985,3.125e-06,388862.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.7628874778747559,0.1295449286699295,0.30000001192092896,0.17597654461860657,0.3499999940395355,0.19674775004386902,0.11288750171661377,0.073255755007267,0.7628874778747559,0.1295449435710907,0.0,1.1113726259281975e-06,2.0979605324100703e-05,0.9999922513961792,1.0000004768371582,1.0000211000442505,9.972968496185786e-05,0.0,0.0,0.0,0.0,0.0,18.99697282537818,0.7777777777777778,,,,,
15,0.0,0.005312301218509674,2.5e-06,416636.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.7899656295776367,0.1296130269765854,0.3499999940395355,0.19674775004386902,0.3500000238418579,0.1967477649450302,0.08996562659740448,0.05463240668177605,0.7899656295776367,0.12961304187774658,0.0,4.004845322924666e-06,4.589592936099507e-05,0.9999337792396545,0.9999885559082031,1.0000029802322388,0.0001807671503684105,0.0,0.0,0.0,0.0,0.0,19.184648096561432,0.8333333333333334,,,,,
16,-0.0,0.005995223298668861,1.8750000000000003e-06,444544.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.789996862411499,0.12048782408237457,0.375,0.20160646736621857,0.32039374113082886,0.18316024541854858,0.09460312128067017,0.062435995787382126,0.789996862411499,0.12048781663179398,0.0,4.866625204158481e-06,2.9087463190080598e-05,0.999959409236908,0.9999855160713196,1.0000004768371582,0.00019320984370096994,0.0,0.0,0.0,0.0,0.0,19.476148523390293,0.8888888888888888,,,,,
17,0.0,0.011378168128430843,1.25e-06,472481.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.8168656826019287,0.09262391924858093,0.375,0.20160646736621857,0.3498094081878662,0.19690066576004028,0.09205625206232071,0.07046373933553696,0.8168656826019287,0.09262390434741974,0.0,5.950785180175444e-06,4.7328881919384e-05,0.9999284744262695,0.9999822974205017,1.000001072883606,0.00021975090658088448,0.0,0.0,0.0,0.0,0.0,19.344543006271124,0.9444444444444444,,,,,
18,0.0,0.018683306872844696,6.25e-07,500266.0,3.0,3.0,3.0,0.0,3.0,3.0,3.0,0.7750625014305115,0.14084643125534058,0.3500000238418579,0.1967477649450302,0.32499998807907104,0.18837164342403412,0.10006250441074371,0.05640558898448944,0.7750625014305115,0.14084644615650177,0.0625,1.1303089195280336e-05,0.00027322862297296524,0.999584436416626,0.9999662637710571,0.9999997615814209,0.000320568448614722,0.0,0.0,0.0,0.0,0.0,17.66909484937787,1.0,,,,,
18,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,1.0,406.352,0.738,0.044,0.0,0.016376476217475202
1 step loss grad_norm learning_rate num_tokens completions/mean_length completions/min_length completions/max_length completions/clipped_ratio completions/mean_terminated_length completions/min_terminated_length completions/max_terminated_length rewards/reward_total/mean rewards/reward_total/std rewards/reward_market/mean rewards/reward_market/std rewards/reward_warehouse/mean rewards/reward_warehouse/std rewards/reward_showroom/mean rewards/reward_showroom/std reward reward_std frac_reward_zero_std sampling/sampling_logp_difference/mean sampling/sampling_logp_difference/max sampling/importance_sampling_ratio/min sampling/importance_sampling_ratio/mean sampling/importance_sampling_ratio/max entropy clip_ratio/low_mean clip_ratio/low_min clip_ratio/high_mean clip_ratio/high_max clip_ratio/region_mean step_time epoch train_runtime train_samples_per_second train_steps_per_second total_flos train_loss
2 1 -0.009 12.235088348388672 0.0 27758.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.7793656587600708 0.12745577096939087 0.25 0.13440430164337158 0.42500001192092896 0.20160645246505737 0.10436563193798065 0.069697305560112 0.7793656587600708 0.12745577096939087 0.0 0.005470390431582928 0.22214925289154053 0.8007970452308655 0.9949374794960022 1.076386570930481 0.028439456821217846 0.0 0.0 0.0 0.0 0.0 19.228480510413647 0.05555555555555555
3 2 0.0723 60.551937103271484 5.000000000000001e-07 55324.0 3.25 3.0 7.0 0.0 3.25 3.0 7.0 0.719434380531311 0.1505809724330902 0.30000001192092896 0.17597654461860657 0.30000001192092896 0.17597654461860657 0.11943437159061432 0.07055392116308212 0.719434380531311 0.1505809724330902 0.0 0.01085888221859932 0.28092825412750244 0.8382877111434937 1.0260276794433594 1.324359655380249 0.036137547835437545 0.0 0.0 0.0 0.0 0.0 17.92062332853675 0.1111111111111111
4 3 0.1153 232.934814453125 1.0000000000000002e-06 83000.0 3.5 3.0 7.0 0.0 3.5 3.0 7.0 0.7885687351226807 0.1279384195804596 0.32500001788139343 0.18837162852287292 0.375 0.20160646736621857 0.08856874704360962 0.07010025531053543 0.7885687351226807 0.1279384344816208 0.0 0.0251829382032156 0.6850378513336182 0.5226751565933228 1.0358223915100098 1.9838452339172363 0.03248842835137111 0.0 0.0 0.0 0.0 0.0 17.184778176248074 0.16666666666666666
5 4 0.0243 13.620709419250488 1.5e-06 110708.0 3.125 3.0 7.0 0.0 3.125 3.0 7.0 0.7762374877929688 0.13016164302825928 0.32499998807907104 0.18837164342403412 0.3500000238418579 0.1967477649450302 0.10123749077320099 0.06825561076402664 0.7762374877929688 0.13016162812709808 0.0 0.007013080175966024 0.2646750509738922 0.7674550414085388 0.9837819337844849 1.0260045528411865 0.03743034108288157 0.0 0.0 0.0 0.0 0.0 17.90316915512085 0.2222222222222222
6 5 0.0004 2.0591225624084473 2.0000000000000003e-06 138452.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.7784374952316284 0.12858590483665466 0.30000001192092896 0.17597654461860657 0.375 0.20160646736621857 0.10343749821186066 0.06380020827054977 0.7784374952316284 0.12858593463897705 0.0 0.001465568202547729 0.05673474818468094 0.9749767780303955 1.0017430782318115 1.058376669883728 0.008188390799773515 0.0 0.0 0.0 0.0 0.0 17.8210555203259 0.2777777777777778
7 6 0.0915 31.611696243286133 2.5e-06 166252.0 3.125 3.0 7.0 0.0 3.125 3.0 7.0 0.7915937900543213 0.12960509955883026 0.375 0.20160646736621857 0.32500001788139343 0.18837162852287292 0.09159374982118607 0.0639258474111557 0.7915937900543213 0.12960509955883026 0.0 0.008641102351248264 0.8236088752746582 0.9935171008110046 1.0399717092514038 2.278707504272461 0.007084679029730978 0.0 0.0 0.0 0.0 0.0 18.056264080107212 0.3333333333333333
8 7 -0.0 0.281630277633667 3e-06 193950.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.7600874900817871 0.13816162943840027 0.32500001788139343 0.18837162852287292 0.32500001788139343 0.18837162852287292 0.11008749902248383 0.07028691470623016 0.7600874900817871 0.13816164433956146 0.0 3.554227441782132e-05 0.0027569520752876997 0.997247576713562 0.9999052286148071 1.0000724792480469 0.0005298306713825696 0.0 0.0 0.0 0.0 0.0 17.43799263238907 0.3888888888888889
9 8 0.0 0.00018881642608903348 3.5e-06 221677.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.7622687816619873 0.1372590959072113 0.4000000059604645 0.20320022106170654 0.25 0.13440430164337158 0.11226874589920044 0.07360353320837021 0.7622687816619873 0.1372590959072113 0.0 2.545601773817907e-07 1.5496943888138048e-06 0.9999986886978149 1.000000238418579 1.0000016689300537 4.1391018498870835e-05 0.0 0.0 0.0 0.0 0.0 17.406394600868225 0.4444444444444444
10 9 0.0 0.00015632262511644512 4.000000000000001e-06 249549.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.775946855545044 0.137176051735878 0.375 0.20160646736621857 0.30000001192092896 0.17597654461860657 0.10094687342643738 0.058497413992881775 0.775946855545044 0.1371760368347168 0.0 2.918131087881193e-07 3.099441755693988e-06 0.9999978542327881 1.0 1.000001311302185 4.173239403826301e-05 0.0 0.0 0.0 0.0 0.0 18.514019537717104 0.5
11 10 0.0 9.833038348006085e-05 4.5e-06 277377.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.7337374687194824 0.15055809915065765 0.2750000059604645 0.1586231142282486 0.3500000238418579 0.1967477649450302 0.10873749852180481 0.06778524816036224 0.7337374687194824 0.15055811405181885 0.0 2.719489771152439e-07 1.9073031580774114e-06 0.9999985694885254 0.9999998807907104 1.0000019073486328 4.224909940830912e-05 0.0 0.0 0.0 0.0 0.0 18.26371632888913 0.5555555555555556
12 11 -0.0 0.00015879125567153096 5e-06 305332.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.7567968368530273 0.13728150725364685 0.375 0.20160646736621857 0.2750000059604645 0.1586231142282486 0.10679687559604645 0.07404065877199173 0.7567968368530273 0.13728150725364685 0.0 3.067227396513772e-07 2.6226434783893637e-06 0.9999973773956299 0.9999994039535522 1.0000019073486328 4.7876037001515215e-05 0.0 0.0 0.0 0.0 0.0 19.4472255371511 0.6111111111111112
13 12 -0.0 0.00039661259506829083 4.3750000000000005e-06 333165.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.7081500291824341 0.1421954482793808 0.30000001192092896 0.17597654461860657 0.2750000059604645 0.1586231142282486 0.13315001130104065 0.07370516657829285 0.7081500291824341 0.142195463180542 0.0 4.768499479723687e-07 5.602954843197949e-06 0.9999943971633911 0.9999988675117493 1.0000016689300537 5.9777358274004655e-05 0.0 0.0 0.0 0.0 0.0 18.45015063509345 0.6666666666666666
14 13 0.0 0.0005282312049530447 3.7500000000000005e-06 361006.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.7762781381607056 0.13823458552360535 0.32500001788139343 0.18837162852287292 0.3500000238418579 0.1967477649450302 0.10127812623977661 0.06158862262964249 0.7762781381607056 0.13823460042476654 0.0 6.830008487668238e-07 5.60290391149465e-06 0.9999943971633911 0.9999983310699463 1.0000022649765015 7.182803437899565e-05 0.0 0.0 0.0 0.0 0.0 17.955969959497452 0.7222222222222222
15 14 -0.0 0.000535853614564985 3.125e-06 388862.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.7628874778747559 0.1295449286699295 0.30000001192092896 0.17597654461860657 0.3499999940395355 0.19674775004386902 0.11288750171661377 0.073255755007267 0.7628874778747559 0.1295449435710907 0.0 1.1113726259281975e-06 2.0979605324100703e-05 0.9999922513961792 1.0000004768371582 1.0000211000442505 9.972968496185786e-05 0.0 0.0 0.0 0.0 0.0 18.99697282537818 0.7777777777777778
16 15 0.0 0.005312301218509674 2.5e-06 416636.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.7899656295776367 0.1296130269765854 0.3499999940395355 0.19674775004386902 0.3500000238418579 0.1967477649450302 0.08996562659740448 0.05463240668177605 0.7899656295776367 0.12961304187774658 0.0 4.004845322924666e-06 4.589592936099507e-05 0.9999337792396545 0.9999885559082031 1.0000029802322388 0.0001807671503684105 0.0 0.0 0.0 0.0 0.0 19.184648096561432 0.8333333333333334
17 16 -0.0 0.005995223298668861 1.8750000000000003e-06 444544.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.789996862411499 0.12048782408237457 0.375 0.20160646736621857 0.32039374113082886 0.18316024541854858 0.09460312128067017 0.062435995787382126 0.789996862411499 0.12048781663179398 0.0 4.866625204158481e-06 2.9087463190080598e-05 0.999959409236908 0.9999855160713196 1.0000004768371582 0.00019320984370096994 0.0 0.0 0.0 0.0 0.0 19.476148523390293 0.8888888888888888
18 17 0.0 0.011378168128430843 1.25e-06 472481.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.8168656826019287 0.09262391924858093 0.375 0.20160646736621857 0.3498094081878662 0.19690066576004028 0.09205625206232071 0.07046373933553696 0.8168656826019287 0.09262390434741974 0.0 5.950785180175444e-06 4.7328881919384e-05 0.9999284744262695 0.9999822974205017 1.000001072883606 0.00021975090658088448 0.0 0.0 0.0 0.0 0.0 19.344543006271124 0.9444444444444444
19 18 0.0 0.018683306872844696 6.25e-07 500266.0 3.0 3.0 3.0 0.0 3.0 3.0 3.0 0.7750625014305115 0.14084643125534058 0.3500000238418579 0.1967477649450302 0.32499998807907104 0.18837164342403412 0.10006250441074371 0.05640558898448944 0.7750625014305115 0.14084644615650177 0.0625 1.1303089195280336e-05 0.00027322862297296524 0.999584436416626 0.9999662637710571 0.9999997615814209 0.000320568448614722 0.0 0.0 0.0 0.0 0.0 17.66909484937787 1.0
20 18 1.0 406.352 0.738 0.044 0.0 0.016376476217475202

695
training_artifacts/metrics.json Normal file
View File

@@ -0,0 +1,695 @@
[
{
"step": 1,
"loss": -0.009,
"grad_norm": 12.235088348388672,
"learning_rate": 0.0,
"num_tokens": 27758.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.7793656587600708,
"rewards/reward_total/std": 0.12745577096939087,
"rewards/reward_market/mean": 0.25,
"rewards/reward_market/std": 0.13440430164337158,
"rewards/reward_warehouse/mean": 0.42500001192092896,
"rewards/reward_warehouse/std": 0.20160645246505737,
"rewards/reward_showroom/mean": 0.10436563193798065,
"rewards/reward_showroom/std": 0.069697305560112,
"reward": 0.7793656587600708,
"reward_std": 0.12745577096939087,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 0.005470390431582928,
"sampling/sampling_logp_difference/max": 0.22214925289154053,
"sampling/importance_sampling_ratio/min": 0.8007970452308655,
"sampling/importance_sampling_ratio/mean": 0.9949374794960022,
"sampling/importance_sampling_ratio/max": 1.076386570930481,
"entropy": 0.028439456821217846,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 19.228480510413647,
"epoch": 0.05555555555555555
},
{
"step": 2,
"loss": 0.0723,
"grad_norm": 60.551937103271484,
"learning_rate": 5.000000000000001e-07,
"num_tokens": 55324.0,
"completions/mean_length": 3.25,
"completions/min_length": 3.0,
"completions/max_length": 7.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.25,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 7.0,
"rewards/reward_total/mean": 0.719434380531311,
"rewards/reward_total/std": 0.1505809724330902,
"rewards/reward_market/mean": 0.30000001192092896,
"rewards/reward_market/std": 0.17597654461860657,
"rewards/reward_warehouse/mean": 0.30000001192092896,
"rewards/reward_warehouse/std": 0.17597654461860657,
"rewards/reward_showroom/mean": 0.11943437159061432,
"rewards/reward_showroom/std": 0.07055392116308212,
"reward": 0.719434380531311,
"reward_std": 0.1505809724330902,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 0.01085888221859932,
"sampling/sampling_logp_difference/max": 0.28092825412750244,
"sampling/importance_sampling_ratio/min": 0.8382877111434937,
"sampling/importance_sampling_ratio/mean": 1.0260276794433594,
"sampling/importance_sampling_ratio/max": 1.324359655380249,
"entropy": 0.036137547835437545,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 17.92062332853675,
"epoch": 0.1111111111111111
},
{
"step": 3,
"loss": 0.1153,
"grad_norm": 232.934814453125,
"learning_rate": 1.0000000000000002e-06,
"num_tokens": 83000.0,
"completions/mean_length": 3.5,
"completions/min_length": 3.0,
"completions/max_length": 7.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.5,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 7.0,
"rewards/reward_total/mean": 0.7885687351226807,
"rewards/reward_total/std": 0.1279384195804596,
"rewards/reward_market/mean": 0.32500001788139343,
"rewards/reward_market/std": 0.18837162852287292,
"rewards/reward_warehouse/mean": 0.375,
"rewards/reward_warehouse/std": 0.20160646736621857,
"rewards/reward_showroom/mean": 0.08856874704360962,
"rewards/reward_showroom/std": 0.07010025531053543,
"reward": 0.7885687351226807,
"reward_std": 0.1279384344816208,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 0.0251829382032156,
"sampling/sampling_logp_difference/max": 0.6850378513336182,
"sampling/importance_sampling_ratio/min": 0.5226751565933228,
"sampling/importance_sampling_ratio/mean": 1.0358223915100098,
"sampling/importance_sampling_ratio/max": 1.9838452339172363,
"entropy": 0.03248842835137111,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 17.184778176248074,
"epoch": 0.16666666666666666
},
{
"step": 4,
"loss": 0.0243,
"grad_norm": 13.620709419250488,
"learning_rate": 1.5e-06,
"num_tokens": 110708.0,
"completions/mean_length": 3.125,
"completions/min_length": 3.0,
"completions/max_length": 7.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.125,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 7.0,
"rewards/reward_total/mean": 0.7762374877929688,
"rewards/reward_total/std": 0.13016164302825928,
"rewards/reward_market/mean": 0.32499998807907104,
"rewards/reward_market/std": 0.18837164342403412,
"rewards/reward_warehouse/mean": 0.3500000238418579,
"rewards/reward_warehouse/std": 0.1967477649450302,
"rewards/reward_showroom/mean": 0.10123749077320099,
"rewards/reward_showroom/std": 0.06825561076402664,
"reward": 0.7762374877929688,
"reward_std": 0.13016162812709808,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 0.007013080175966024,
"sampling/sampling_logp_difference/max": 0.2646750509738922,
"sampling/importance_sampling_ratio/min": 0.7674550414085388,
"sampling/importance_sampling_ratio/mean": 0.9837819337844849,
"sampling/importance_sampling_ratio/max": 1.0260045528411865,
"entropy": 0.03743034108288157,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 17.90316915512085,
"epoch": 0.2222222222222222
},
{
"step": 5,
"loss": 0.0004,
"grad_norm": 2.0591225624084473,
"learning_rate": 2.0000000000000003e-06,
"num_tokens": 138452.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.7784374952316284,
"rewards/reward_total/std": 0.12858590483665466,
"rewards/reward_market/mean": 0.30000001192092896,
"rewards/reward_market/std": 0.17597654461860657,
"rewards/reward_warehouse/mean": 0.375,
"rewards/reward_warehouse/std": 0.20160646736621857,
"rewards/reward_showroom/mean": 0.10343749821186066,
"rewards/reward_showroom/std": 0.06380020827054977,
"reward": 0.7784374952316284,
"reward_std": 0.12858593463897705,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 0.001465568202547729,
"sampling/sampling_logp_difference/max": 0.05673474818468094,
"sampling/importance_sampling_ratio/min": 0.9749767780303955,
"sampling/importance_sampling_ratio/mean": 1.0017430782318115,
"sampling/importance_sampling_ratio/max": 1.058376669883728,
"entropy": 0.008188390799773515,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 17.8210555203259,
"epoch": 0.2777777777777778
},
{
"step": 6,
"loss": 0.0915,
"grad_norm": 31.611696243286133,
"learning_rate": 2.5e-06,
"num_tokens": 166252.0,
"completions/mean_length": 3.125,
"completions/min_length": 3.0,
"completions/max_length": 7.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.125,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 7.0,
"rewards/reward_total/mean": 0.7915937900543213,
"rewards/reward_total/std": 0.12960509955883026,
"rewards/reward_market/mean": 0.375,
"rewards/reward_market/std": 0.20160646736621857,
"rewards/reward_warehouse/mean": 0.32500001788139343,
"rewards/reward_warehouse/std": 0.18837162852287292,
"rewards/reward_showroom/mean": 0.09159374982118607,
"rewards/reward_showroom/std": 0.0639258474111557,
"reward": 0.7915937900543213,
"reward_std": 0.12960509955883026,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 0.008641102351248264,
"sampling/sampling_logp_difference/max": 0.8236088752746582,
"sampling/importance_sampling_ratio/min": 0.9935171008110046,
"sampling/importance_sampling_ratio/mean": 1.0399717092514038,
"sampling/importance_sampling_ratio/max": 2.278707504272461,
"entropy": 0.007084679029730978,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 18.056264080107212,
"epoch": 0.3333333333333333
},
{
"step": 7,
"loss": -0.0,
"grad_norm": 0.281630277633667,
"learning_rate": 3e-06,
"num_tokens": 193950.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.7600874900817871,
"rewards/reward_total/std": 0.13816162943840027,
"rewards/reward_market/mean": 0.32500001788139343,
"rewards/reward_market/std": 0.18837162852287292,
"rewards/reward_warehouse/mean": 0.32500001788139343,
"rewards/reward_warehouse/std": 0.18837162852287292,
"rewards/reward_showroom/mean": 0.11008749902248383,
"rewards/reward_showroom/std": 0.07028691470623016,
"reward": 0.7600874900817871,
"reward_std": 0.13816164433956146,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 3.554227441782132e-05,
"sampling/sampling_logp_difference/max": 0.0027569520752876997,
"sampling/importance_sampling_ratio/min": 0.997247576713562,
"sampling/importance_sampling_ratio/mean": 0.9999052286148071,
"sampling/importance_sampling_ratio/max": 1.0000724792480469,
"entropy": 0.0005298306713825696,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 17.43799263238907,
"epoch": 0.3888888888888889
},
{
"step": 8,
"loss": 0.0,
"grad_norm": 0.00018881642608903348,
"learning_rate": 3.5e-06,
"num_tokens": 221677.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.7622687816619873,
"rewards/reward_total/std": 0.1372590959072113,
"rewards/reward_market/mean": 0.4000000059604645,
"rewards/reward_market/std": 0.20320022106170654,
"rewards/reward_warehouse/mean": 0.25,
"rewards/reward_warehouse/std": 0.13440430164337158,
"rewards/reward_showroom/mean": 0.11226874589920044,
"rewards/reward_showroom/std": 0.07360353320837021,
"reward": 0.7622687816619873,
"reward_std": 0.1372590959072113,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 2.545601773817907e-07,
"sampling/sampling_logp_difference/max": 1.5496943888138048e-06,
"sampling/importance_sampling_ratio/min": 0.9999986886978149,
"sampling/importance_sampling_ratio/mean": 1.000000238418579,
"sampling/importance_sampling_ratio/max": 1.0000016689300537,
"entropy": 4.1391018498870835e-05,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 17.406394600868225,
"epoch": 0.4444444444444444
},
{
"step": 9,
"loss": 0.0,
"grad_norm": 0.00015632262511644512,
"learning_rate": 4.000000000000001e-06,
"num_tokens": 249549.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.775946855545044,
"rewards/reward_total/std": 0.137176051735878,
"rewards/reward_market/mean": 0.375,
"rewards/reward_market/std": 0.20160646736621857,
"rewards/reward_warehouse/mean": 0.30000001192092896,
"rewards/reward_warehouse/std": 0.17597654461860657,
"rewards/reward_showroom/mean": 0.10094687342643738,
"rewards/reward_showroom/std": 0.058497413992881775,
"reward": 0.775946855545044,
"reward_std": 0.1371760368347168,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 2.918131087881193e-07,
"sampling/sampling_logp_difference/max": 3.099441755693988e-06,
"sampling/importance_sampling_ratio/min": 0.9999978542327881,
"sampling/importance_sampling_ratio/mean": 1.0,
"sampling/importance_sampling_ratio/max": 1.000001311302185,
"entropy": 4.173239403826301e-05,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 18.514019537717104,
"epoch": 0.5
},
{
"step": 10,
"loss": 0.0,
"grad_norm": 9.833038348006085e-05,
"learning_rate": 4.5e-06,
"num_tokens": 277377.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.7337374687194824,
"rewards/reward_total/std": 0.15055809915065765,
"rewards/reward_market/mean": 0.2750000059604645,
"rewards/reward_market/std": 0.1586231142282486,
"rewards/reward_warehouse/mean": 0.3500000238418579,
"rewards/reward_warehouse/std": 0.1967477649450302,
"rewards/reward_showroom/mean": 0.10873749852180481,
"rewards/reward_showroom/std": 0.06778524816036224,
"reward": 0.7337374687194824,
"reward_std": 0.15055811405181885,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 2.719489771152439e-07,
"sampling/sampling_logp_difference/max": 1.9073031580774114e-06,
"sampling/importance_sampling_ratio/min": 0.9999985694885254,
"sampling/importance_sampling_ratio/mean": 0.9999998807907104,
"sampling/importance_sampling_ratio/max": 1.0000019073486328,
"entropy": 4.224909940830912e-05,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 18.26371632888913,
"epoch": 0.5555555555555556
},
{
"step": 11,
"loss": -0.0,
"grad_norm": 0.00015879125567153096,
"learning_rate": 5e-06,
"num_tokens": 305332.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.7567968368530273,
"rewards/reward_total/std": 0.13728150725364685,
"rewards/reward_market/mean": 0.375,
"rewards/reward_market/std": 0.20160646736621857,
"rewards/reward_warehouse/mean": 0.2750000059604645,
"rewards/reward_warehouse/std": 0.1586231142282486,
"rewards/reward_showroom/mean": 0.10679687559604645,
"rewards/reward_showroom/std": 0.07404065877199173,
"reward": 0.7567968368530273,
"reward_std": 0.13728150725364685,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 3.067227396513772e-07,
"sampling/sampling_logp_difference/max": 2.6226434783893637e-06,
"sampling/importance_sampling_ratio/min": 0.9999973773956299,
"sampling/importance_sampling_ratio/mean": 0.9999994039535522,
"sampling/importance_sampling_ratio/max": 1.0000019073486328,
"entropy": 4.7876037001515215e-05,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 19.4472255371511,
"epoch": 0.6111111111111112
},
{
"step": 12,
"loss": -0.0,
"grad_norm": 0.00039661259506829083,
"learning_rate": 4.3750000000000005e-06,
"num_tokens": 333165.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.7081500291824341,
"rewards/reward_total/std": 0.1421954482793808,
"rewards/reward_market/mean": 0.30000001192092896,
"rewards/reward_market/std": 0.17597654461860657,
"rewards/reward_warehouse/mean": 0.2750000059604645,
"rewards/reward_warehouse/std": 0.1586231142282486,
"rewards/reward_showroom/mean": 0.13315001130104065,
"rewards/reward_showroom/std": 0.07370516657829285,
"reward": 0.7081500291824341,
"reward_std": 0.142195463180542,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 4.768499479723687e-07,
"sampling/sampling_logp_difference/max": 5.602954843197949e-06,
"sampling/importance_sampling_ratio/min": 0.9999943971633911,
"sampling/importance_sampling_ratio/mean": 0.9999988675117493,
"sampling/importance_sampling_ratio/max": 1.0000016689300537,
"entropy": 5.9777358274004655e-05,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 18.45015063509345,
"epoch": 0.6666666666666666
},
{
"step": 13,
"loss": 0.0,
"grad_norm": 0.0005282312049530447,
"learning_rate": 3.7500000000000005e-06,
"num_tokens": 361006.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.7762781381607056,
"rewards/reward_total/std": 0.13823458552360535,
"rewards/reward_market/mean": 0.32500001788139343,
"rewards/reward_market/std": 0.18837162852287292,
"rewards/reward_warehouse/mean": 0.3500000238418579,
"rewards/reward_warehouse/std": 0.1967477649450302,
"rewards/reward_showroom/mean": 0.10127812623977661,
"rewards/reward_showroom/std": 0.06158862262964249,
"reward": 0.7762781381607056,
"reward_std": 0.13823460042476654,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 6.830008487668238e-07,
"sampling/sampling_logp_difference/max": 5.60290391149465e-06,
"sampling/importance_sampling_ratio/min": 0.9999943971633911,
"sampling/importance_sampling_ratio/mean": 0.9999983310699463,
"sampling/importance_sampling_ratio/max": 1.0000022649765015,
"entropy": 7.182803437899565e-05,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 17.955969959497452,
"epoch": 0.7222222222222222
},
{
"step": 14,
"loss": -0.0,
"grad_norm": 0.000535853614564985,
"learning_rate": 3.125e-06,
"num_tokens": 388862.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.7628874778747559,
"rewards/reward_total/std": 0.1295449286699295,
"rewards/reward_market/mean": 0.30000001192092896,
"rewards/reward_market/std": 0.17597654461860657,
"rewards/reward_warehouse/mean": 0.3499999940395355,
"rewards/reward_warehouse/std": 0.19674775004386902,
"rewards/reward_showroom/mean": 0.11288750171661377,
"rewards/reward_showroom/std": 0.073255755007267,
"reward": 0.7628874778747559,
"reward_std": 0.1295449435710907,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 1.1113726259281975e-06,
"sampling/sampling_logp_difference/max": 2.0979605324100703e-05,
"sampling/importance_sampling_ratio/min": 0.9999922513961792,
"sampling/importance_sampling_ratio/mean": 1.0000004768371582,
"sampling/importance_sampling_ratio/max": 1.0000211000442505,
"entropy": 9.972968496185786e-05,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 18.99697282537818,
"epoch": 0.7777777777777778
},
{
"step": 15,
"loss": 0.0,
"grad_norm": 0.005312301218509674,
"learning_rate": 2.5e-06,
"num_tokens": 416636.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.7899656295776367,
"rewards/reward_total/std": 0.1296130269765854,
"rewards/reward_market/mean": 0.3499999940395355,
"rewards/reward_market/std": 0.19674775004386902,
"rewards/reward_warehouse/mean": 0.3500000238418579,
"rewards/reward_warehouse/std": 0.1967477649450302,
"rewards/reward_showroom/mean": 0.08996562659740448,
"rewards/reward_showroom/std": 0.05463240668177605,
"reward": 0.7899656295776367,
"reward_std": 0.12961304187774658,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 4.004845322924666e-06,
"sampling/sampling_logp_difference/max": 4.589592936099507e-05,
"sampling/importance_sampling_ratio/min": 0.9999337792396545,
"sampling/importance_sampling_ratio/mean": 0.9999885559082031,
"sampling/importance_sampling_ratio/max": 1.0000029802322388,
"entropy": 0.0001807671503684105,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 19.184648096561432,
"epoch": 0.8333333333333334
},
{
"step": 16,
"loss": -0.0,
"grad_norm": 0.005995223298668861,
"learning_rate": 1.8750000000000003e-06,
"num_tokens": 444544.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.789996862411499,
"rewards/reward_total/std": 0.12048782408237457,
"rewards/reward_market/mean": 0.375,
"rewards/reward_market/std": 0.20160646736621857,
"rewards/reward_warehouse/mean": 0.32039374113082886,
"rewards/reward_warehouse/std": 0.18316024541854858,
"rewards/reward_showroom/mean": 0.09460312128067017,
"rewards/reward_showroom/std": 0.062435995787382126,
"reward": 0.789996862411499,
"reward_std": 0.12048781663179398,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 4.866625204158481e-06,
"sampling/sampling_logp_difference/max": 2.9087463190080598e-05,
"sampling/importance_sampling_ratio/min": 0.999959409236908,
"sampling/importance_sampling_ratio/mean": 0.9999855160713196,
"sampling/importance_sampling_ratio/max": 1.0000004768371582,
"entropy": 0.00019320984370096994,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 19.476148523390293,
"epoch": 0.8888888888888888
},
{
"step": 17,
"loss": 0.0,
"grad_norm": 0.011378168128430843,
"learning_rate": 1.25e-06,
"num_tokens": 472481.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.8168656826019287,
"rewards/reward_total/std": 0.09262391924858093,
"rewards/reward_market/mean": 0.375,
"rewards/reward_market/std": 0.20160646736621857,
"rewards/reward_warehouse/mean": 0.3498094081878662,
"rewards/reward_warehouse/std": 0.19690066576004028,
"rewards/reward_showroom/mean": 0.09205625206232071,
"rewards/reward_showroom/std": 0.07046373933553696,
"reward": 0.8168656826019287,
"reward_std": 0.09262390434741974,
"frac_reward_zero_std": 0.0,
"sampling/sampling_logp_difference/mean": 5.950785180175444e-06,
"sampling/sampling_logp_difference/max": 4.7328881919384e-05,
"sampling/importance_sampling_ratio/min": 0.9999284744262695,
"sampling/importance_sampling_ratio/mean": 0.9999822974205017,
"sampling/importance_sampling_ratio/max": 1.000001072883606,
"entropy": 0.00021975090658088448,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 19.344543006271124,
"epoch": 0.9444444444444444
},
{
"step": 18,
"loss": 0.0,
"grad_norm": 0.018683306872844696,
"learning_rate": 6.25e-07,
"num_tokens": 500266.0,
"completions/mean_length": 3.0,
"completions/min_length": 3.0,
"completions/max_length": 3.0,
"completions/clipped_ratio": 0.0,
"completions/mean_terminated_length": 3.0,
"completions/min_terminated_length": 3.0,
"completions/max_terminated_length": 3.0,
"rewards/reward_total/mean": 0.7750625014305115,
"rewards/reward_total/std": 0.14084643125534058,
"rewards/reward_market/mean": 0.3500000238418579,
"rewards/reward_market/std": 0.1967477649450302,
"rewards/reward_warehouse/mean": 0.32499998807907104,
"rewards/reward_warehouse/std": 0.18837164342403412,
"rewards/reward_showroom/mean": 0.10006250441074371,
"rewards/reward_showroom/std": 0.05640558898448944,
"reward": 0.7750625014305115,
"reward_std": 0.14084644615650177,
"frac_reward_zero_std": 0.0625,
"sampling/sampling_logp_difference/mean": 1.1303089195280336e-05,
"sampling/sampling_logp_difference/max": 0.00027322862297296524,
"sampling/importance_sampling_ratio/min": 0.999584436416626,
"sampling/importance_sampling_ratio/mean": 0.9999662637710571,
"sampling/importance_sampling_ratio/max": 0.9999997615814209,
"entropy": 0.000320568448614722,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/high_max": 0.0,
"clip_ratio/region_mean": 0.0,
"step_time": 17.66909484937787,
"epoch": 1.0
},
{
"step": 18,
"train_runtime": 406.352,
"train_samples_per_second": 0.738,
"train_steps_per_second": 0.044,
"total_flos": 0.0,
"train_loss": 0.016376476217475202,
"epoch": 1.0
}
]

BIN
training_artifacts/reward_curve.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 55 KiB

BIN
training_artifacts/reward_total_curve.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 56 KiB

63
training_artifacts/training_summary.json Normal file
View File

@@ -0,0 +1,63 @@
{
"loss": {
"final": 0.0,
"max": 0.1153,
"min": -0.009,
"mean": 0.01637777777777778,
"n": 18
},
"reward_total": {
"final": 0.7750625014305115,
"max": 0.8168656826019287,
"min": 0.7081500291824341,
"mean": 0.7689822945329878,
"n": 18
},
"reward_market": {
"final": 0.0,
"max": 0.0,
"min": 0.0,
"mean": 0.0,
"n": 0
},
"reward_warehouse": {
"final": 0.0,
"max": 0.0,
"min": 0.0,
"mean": 0.0,
"n": 0
},
"reward_showroom": {
"final": 0.0,
"max": 0.0,
"min": 0.0,
"mean": 0.0,
"n": 0
},
"n_log_rows": 19,
"output_dir": "/workspace/shopmanager-grpo-qwen3",
"run_config": {
"model": "Qwen/Qwen3-1.7B",
"env_url": "https://hard007ik-shopmanagereng.hf.space",
"dataset_size": 300,
"num_generations": 2,
"per_device_batch": 1,
"grad_accum": 32,
"max_completion_length": 64,
"max_turns": 15,
"lr": 5e-06,
"warmup_steps": 10,
"max_steps": -1,
"epochs": 1,
"vllm_gpu_mem": 0.3,
"reward_weights": [
1.0,
0.0,
0.0,
0.0
],
"precision": {
"bf16": true
}
}
}