16 lines
11 KiB
CSV
16 lines
11 KiB
CSV
|
async/discard_rate,async/discarded_count,async/effective_batch_groups,async/effective_batch_samples,async/staleness_max,async/staleness_mean,async/staleness_min,async/staleness_ratio,generate/avg_num_tokens,generate/avg_tokens_non_zero_rewards,generate/avg_tokens_zero_rewards,generate/max_num_tokens,generate/min_num_tokens,generate/std_num_tokens,generate/tis/aligned_tokens,generate/tis/alignment_fail_count,generate/tis/exact_match_fraction,generate/tis/lcs_fallback_fraction,generate/tis/lcs_fallback_messages,generate/tis/unaligned_fraction,loss/avg_final_rewards,loss/avg_raw_advantages,loss/avg_raw_advantages_abs,policy/final_loss,policy/log_ratio_abs_max,policy/log_ratio_abs_mean,policy/log_ratio_abs_p99,policy/log_ratio_abs_pos00,policy/log_ratio_abs_pos10,policy/log_ratio_abs_pos20,policy/log_ratio_abs_pos30,policy/log_ratio_abs_pos40,policy/log_ratio_abs_pos50,policy/log_ratio_abs_pos60,policy/log_ratio_abs_pos70,policy/log_ratio_abs_pos80,policy/log_ratio_abs_pos90,policy/n_tokens_dp_gt_10pct,policy/n_tokens_dp_gt_1pct,policy/n_tokens_dp_gt_50pct,policy/policy_entropy,policy/policy_loss,policy/policy_lr,policy/policy_update_steps,policy/ppo_clip_ratio,policy/raw_grad_norm,policy/rollout_train_prob_diff_mean,policy/rollout_train_prob_diff_std,policy/tis/imp_ratio_capped_fraction,policy/tis/imp_ratio_mean,policy/tis/log_ratio_abs_mean,reward/avg_pass_at_8,reward/avg_raw_reward,system/process_rss_gb,system/process_vms_gb,system/ram_available_gb,system/ram_percent,system/ram_total_gb,system/ram_used_gb,timing/compute_advantages_and_returns,timing/convert_to_training_input,timing/fwd_logprobs_values_reward,timing/policy_train,timing/run_training,timing/step,timing/sync_weights,timing/train_critic_and_policy,timing/wait_for_generation_buffer,tis/batch_skipped_no_logprobs,tis/skipped_fraction,trainer/epoch,trainer/global_step,batch_errors/total_batches,batch_errors/total_instances,batch_errors/total_successful,batch_errors/total_failed,batch_errors/total_masked,batch_errors/avg_VerifierTimeoutError,batch_errors/total_VerifierTimeoutError,batch_errors/avg_AgentTimeoutError,batch_errors/total_AgentTimeoutError,batch_errors/avg_AgentSetupTimeoutError,batch_errors/total_AgentSetupTimeoutError,batch_errors/avg_InvalidChatHistory,batch_errors/total_InvalidChatHistory,batch_errors/avg_ContextLengthExceededError,batch_errors/total_ContextLengthExceededError,batch_errors/avg_DaytonaAuthenticationError,batch_errors/total_DaytonaAuthenticationError,timing/cleanup_old_checkpoints,timing/save_checkpoints,batch_errors/avg_Timeout,batch_errors/total_Timeout,timing/save_hf_model,batch_errors/avg_DaytonaError,batch_errors/total_DaytonaError
|
||
|
|
0.0,0,64,512,0,0.0,0,0.0,1665.6621,3021.6782,967.5947,14153,1,2057.7122,573938.0,33.0,0.9487,0.0,0.0,0.0513,0.3398,0.2099,0.2501,-0.0019,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0656,-0.1214,0.0,1.0,0.0,0.0309,425844310016.0,310867954499584.0,0.0,0.5258,0.022,0.6406,0.3398,7.5378,49.9637,630.257,26.5,857.9676,227.7106,0.0376,2.4443,199.3619,89.3179,288.8421,11028.1246,18.1382,89.4422,10718.6996,0.0,0.0,0,1,121,968,497,114,216,1.9090909090909092,231,1.6528925619834711,200.0,0.2231404958677686,27.0,0.15702479338842976,19,0.09090909090909091,11.0,0.024793388429752067,3.0,,,,,,,
|
||
|
|
0.0,0,64,512,1,1.0,1,1.0,2192.3809,5107.972,1422.0889,31541,1,3242.089,778056.0,90.0,0.8463,0.0,0.0,0.1537,0.209,0.14,0.236,-0.0009,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0639,-0.0569,0.0,1.0,0.0,0.0223,5135849414983680.0,3.8306663504564716e+18,0.0001,0.5252,0.0393,0.4844,0.209,13.664,50.6382,619.7108,27.8,857.9676,238.2568,0.0608,5.2456,18.5674,134.9773,153.7971,3675.2048,21.6594,135.1685,3494.5009,0.0,0.0,0,2,64,512,253,62,90,2.109375,135,1.25,80.0,0.5,32.0,0.109375,7,0.046875,3.0,0.03125,2.0,139.9582,149.9938,,,,,
|
||
|
|
0.0,0,64,512,2,1.6094,0,0.9844,1974.7832,3706.1466,1467.6162,19867,1,2776.8866,687353.0,78.0,0.904,0.0,0.0,0.096,0.2266,0.1618,0.2338,-0.0013,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0563,-0.0823,0.0,1.0,0.0,0.0252,5712403677315072.0,4.576328473257181e+18,0.0,0.4977,0.0341,0.5156,0.2266,13.514,50.458,618.5668,27.9,857.9676,239.4008,0.0426,3.6209,18.1097,113.5752,131.9041,3304.1227,19.8368,113.7514,3148.7583,0.0,0.0,0,3,63,504,278,59,93,1.7619047619047619,111,1.3015873015873016,82.0,0.31746031746031744,20.0,0.14285714285714285,9,0.031746031746031744,2.0,0.015873015873015872,1.0,,,0.015873015873015872,1.0,,,
|
||
|
|
0.0,0,64,512,3,1.7656,0,0.9375,1916.1016,3570.8227,1287.2183,26360,1,2567.8385,672216.0,119.0,0.8943,0.0,0.0,0.1057,0.2754,0.1458,0.2335,-0.0013,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0609,-0.081,0.0,1.0,0.0,0.0264,2.5092910058202726e+17,2.0297533524949546e+20,0.0,0.5539,0.0386,0.5312,0.2754,13.7533,50.7891,617.0518,28.1,857.9676,240.9158,0.0533,4.4556,16.0924,108.7123,125.0416,3238.4887,20.0299,108.8955,3088.9592,0.0,0.0,0,4,59,472,401,29,13,1.0,59,0.15254237288135594,9.0,0.0847457627118644,5.0,0.01694915254237288,1,0.06779661016949153,4.0,,,0.0083,12.9332,,,,,
|
||
|
|
0.0,0,64,512,3,1.4531,0,0.9219,3224.291,3590.6844,2890.709,13914,1,2397.9485,1106110.0,103.0,0.9644,0.0,0.0,0.0356,0.4766,0.064,0.1543,-0.0006,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0788,-0.0412,0.0,1.0,0.0,0.017,2.1654929612682035e+18,2.119323756633866e+21,0.0,0.8649,0.0405,0.625,0.4766,13.7164,50.7537,614.8819,28.3,857.9676,243.0857,0.0323,2.8211,19.9051,118.6808,138.8115,1062.3118,23.3888,118.8737,897.2872,0.0,0.0,0,5,63,504,460,23,14,0.42857142857142855,27,0.06349206349206349,4.0,0.047619047619047616,3.0,0.09523809523809523,6,0.047619047619047616,3.0,,,,,,,16.893,0.015873015873015872,1.0
|
||
|
|
0.0,0,64,512,4,1.6406,0,0.8594,4239.8125,4633.2308,3790.4268,22031,1,3300.5798,1457215.0,112.0,0.9654,0.0,0.0,0.0346,0.5332,0.044,0.102,-0.0004,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0853,-0.025,0.0,1.0,0.0,0.0138,6.141298557442851e+18,5.070549340210866e+21,0.0,0.9245,0.0456,0.6562,0.5332,14.4743,51.5385,613.4079,28.5,857.9676,244.5597,0.0452,4.2242,25.388,159.3519,185.0213,1290.7407,28.6557,159.5877,1072.8381,0.0,0.0,0,6,58,464,445,11,12,0.1206896551724138,7,,,,,0.10344827586206896,6,0.034482758620689655,2.0,,,0.0083,10.3463,,,,0.06896551724137931,4.0
|
||
|
|
0.0,0,64,512,4,1.8906,0,0.8906,4397.6465,4166.2911,4597.0327,25556,1,3231.8595,1555192.0,95.0,0.9507,0.0,0.0,0.0493,0.4629,0.0201,0.1187,-0.0001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0913,-0.0045,0.0,1.0,0.0,0.0167,4.5730908733082645e+20,5.0551306138646176e+23,0.0,0.9745,0.0461,0.5781,0.4629,14.6233,51.6414,612.7198,28.6,857.9676,245.2477,0.0504,4.7036,26.7455,163.7089,190.7489,2018.6125,34.5773,163.9526,1788.5789,0.0,0.0,0,7,60,480,458,10,12,0.15,9,,,0.016666666666666666,1.0,0.16666666666666666,10,0.016666666666666666,1.0,,,,,,,,0.016666666666666666,1.0
|
||
|
|
0.0,0,64,512,4,2.1406,0,0.9531,5137.9238,4980.2167,5241.5307,31761,1,4015.1526,1850673.0,87.0,0.9135,0.0,0.0,0.0865,0.3965,0.0072,0.0726,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0938,0.0003,0.0,1.0,0.0,0.0119,2.930747012383905e+21,3.2028701271085374e+24,0.0,0.9696,0.046,0.4688,0.3965,15.9055,52.3258,610.4556,28.8,857.9676,247.512,0.0614,6.5073,32.8918,190.3358,223.5354,1224.5185,28.8017,190.5818,965.6712,0.0,0.0,0,8,63,504,480,13,21,0.047619047619047616,3,0.07936507936507936,5.0,,,0.20634920634920634,13,,,,,0.0081,12.67,,,,0.047619047619047616,3.0
|
||
|
|
0.0,0,64,512,5,1.9219,0,0.9062,4859.3164,4603.0995,5038.9236,31797,1,4029.8322,1739932.0,52.0,0.8962,0.0,0.0,0.1038,0.4121,0.0093,0.1242,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0803,0.0001,0.0,1.0,0.0,0.0145,1.7394266914541902e+21,2.2925263334899206e+24,0.0,0.9646,0.0375,0.5469,0.4121,16.1924,52.5256,608.6286,29.1,857.9676,249.339,0.0627,5.7252,31.0932,188.5143,219.9568,1481.3095,32.107,188.8005,1223.5171,0.0,0.0,0,9,63,504,487,10,14,0.047619047619047616,3,0.015873015873015872,1.0,,,0.1746031746031746,11,,,,,,,,,,0.031746031746031744,2.0
|
||
|
|
0.0,0,64,512,3,1.6562,0,0.8594,4206.2637,3770.6644,4770.7848,31696,1,3098.8536,1466169.0,27.0,0.9684,0.0,0.0,0.0316,0.5645,-0.0016,0.1212,-0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0758,-0.0009,0.0,1.0,0.0,0.0186,2.486790661662018e+22,3.009927560391077e+25,0.0,0.9792,0.0338,0.7031,0.5645,16.1496,52.6445,607.3474,29.2,857.9676,250.6202,0.062,5.807,27.2616,157.0913,184.6351,1344.8255,28.0337,157.3111,1126.3452,0.0,0.0,0,10,62,496,465,10,25,0.0967741935483871,6,0.04838709677419355,3.0,,,0.3387096774193548,21,0.016129032258064516,1.0,,,0.0079,11.2982,,,6.1509,,
|
||
|
|
0.0,0,64,512,4,1.8125,0,0.8594,4341.25,4421.7844,4281.534,31768,1,3837.6661,1544551.0,26.0,0.9519,0.0,0.0,0.0481,0.4258,0.0172,0.1082,-0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0768,-0.0016,0.0,1.0,0.0,0.0145,2.3760572720076554e+18,2.479232720767365e+21,0.0,0.9513,0.0343,0.5312,0.4258,16.1263,52.6095,605.798,29.4,857.9676,252.1696,0.0603,5.7445,30.3901,180.1222,210.8445,1608.5564,32.131,180.3938,1359.8319,0.0,0.0,0,11,63,504,482,8,13,0.14285714285714285,9,0.031746031746031744,2.0,,,0.1746031746031746,11,,,,,,,,,,,
|
||
|
|
0.0,0,64,512,4,2.0156,0,0.9375,5171.2402,4363.3762,5697.6548,31752,1,3838.9164,1805076.0,21.0,0.972,0.0,0.0,0.028,0.3945,-0.0056,0.099,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0793,0.0001,0.0,1.0,0.0,0.0187,3.3199502774606234e+17,4.460413625381222e+20,0.0,0.9698,0.0322,0.5156,0.3945,16.5123,53.0591,604.8148,29.5,857.9676,253.1528,0.5752,5.7891,36.193,189.4182,226.4272,1418.9828,24.7579,189.6581,1162.0041,0.0,0.0,0,12,63,504,486,8,16,0.031746031746031744,2,0.1111111111111111,7.0,,,0.14285714285714285,9,,,,,0.0085,10.287,,,,,
|
||
|
|
0.0,0,64,512,3,1.5781,0,0.8594,4488.6309,4142.0406,4878.3651,31794,1,3769.576,1574682.0,18.0,0.937,0.0,0.0,0.063,0.5293,0.0116,0.1159,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0641,0.0001,0.0,1.0,0.0,0.0191,1.1232591437954967e+20,1.4095364312304327e+23,0.0,0.9755,0.0277,0.6719,0.5293,16.3773,52.9352,603.7007,29.6,857.9676,254.2668,0.0652,5.8155,29.0791,176.4044,205.7881,1174.4667,23.8728,176.6434,938.9854,0.0,0.0,0,13,63,504,462,18,34,0.12698412698412698,8,0.12698412698412698,8.0,,,0.36507936507936506,23,0.047619047619047616,3.0,,,,,,,,,
|
||
|
|
0.0,0,64,512,4,1.9688,0,0.9375,4663.1953,4560.4923,4726.3722,31674,1,3835.0487,1647855.0,26.0,0.9488,0.0,0.0,0.0512,0.3809,-0.0175,0.1338,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0707,0.0,0.0,1.0,0.0,0.0173,1.3211500909875513e+22,1.6953347554969734e+25,0.0,0.9296,0.0299,0.5,0.3809,16.7322,53.2489,601.7022,29.9,857.9676,256.2654,0.1012,5.8519,30.3985,178.5781,209.3207,1621.8309,26.8666,178.8196,1379.7871,0.0,0.0,0,14,64,512,491,8,20,0.015625,1,0.1875,12.0,,,0.109375,7,0.015625,1.0,,,0.0082,11.2278,,,,,
|