18 lines
9.0 KiB
CSV
18 lines
9.0 KiB
CSV
|
log_file,async/discard_rate,async/discarded_count,async/effective_batch_groups,async/effective_batch_samples,async/staleness_max,async/staleness_mean,async/staleness_min,async/staleness_ratio,generate/avg_num_tokens,generate/avg_tokens_non_zero_rewards,generate/avg_tokens_zero_rewards,generate/max_num_tokens,generate/min_num_tokens,generate/std_num_tokens,loss/avg_final_rewards,loss/avg_raw_advantages,loss/avg_raw_advantages_abs,policy/final_loss,policy/log_ratio_abs_max,policy/log_ratio_abs_mean,policy/log_ratio_abs_p99,policy/log_ratio_abs_pos00,policy/log_ratio_abs_pos10,policy/log_ratio_abs_pos20,policy/log_ratio_abs_pos30,policy/log_ratio_abs_pos40,policy/log_ratio_abs_pos50,policy/log_ratio_abs_pos60,policy/log_ratio_abs_pos70,policy/log_ratio_abs_pos80,policy/log_ratio_abs_pos90,policy/n_tokens_dp_gt_10pct,policy/n_tokens_dp_gt_1pct,policy/n_tokens_dp_gt_50pct,policy/policy_entropy,policy/policy_loss,policy/policy_lr,policy/policy_update_steps,policy/ppo_clip_ratio,policy/raw_grad_norm,reward/avg_pass_at_8,reward/avg_raw_reward,system/process_rss_gb,system/process_vms_gb,system/ram_available_gb,system/ram_percent,system/ram_total_gb,system/ram_used_gb,timing/compute_advantages_and_returns,timing/convert_to_training_input,timing/fwd_logprobs_values_reward,timing/policy_train,timing/run_training,timing/step,timing/sync_weights,timing/train_critic_and_policy,timing/wait_for_generation_buffer,trainer/epoch,trainer/global_step,batch_errors/total_batches,batch_errors/total_instances,batch_errors/total_successful,batch_errors/total_failed,batch_errors/total_masked,batch_errors/avg_ContextLengthExceededError,batch_errors/total_ContextLengthExceededError,timing/cleanup_old_checkpoints,timing/save_checkpoints,batch_errors/avg_DaytonaError,batch_errors/total_DaytonaError,batch_errors/avg_VerifierTimeoutError,batch_errors/total_VerifierTimeoutError,batch_errors/avg_InvalidChatHistory,batch_errors/total_InvalidChatHistory,batch_errors/avg_AgentTimeoutError,batch_errors/total_AgentTimeoutError,timing/save_hf_model,batch_errors/avg_RuntimeError,batch_errors/total_RuntimeError,global_step
|
||
|
|
job_591150,0.0,0,64,512,0,0.0,0,0.0,8795.541,7557.8707,9821.0393,28234,1862,5511.4955,0.4531,0.0154,0.1383,-0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1147,-0.0,0.0,1.0,0.0,0.0154,0.6094,0.4531,12.2192,50.7338,643.6898,25.0,857.9676,214.2778,0.1433,4.903,67.5598,331.9725,399.8923,2717.3975,19.1362,332.1889,2293.4656,0,1,127.0,1016.0,978.0,16.0,38.0,0.31496062992125984,40.0,,,,,,,,,,,,,,1
|
||
|
|
job_591150,0.0,0,64,512,1,0.8906,0,0.8906,10265.4648,9975.6566,10448.2102,30188,1,7251.4743,0.3867,0.0046,0.1482,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1058,0.0,0.0,1.0,0.0,0.0194,0.5312,0.3867,12.8623,51.2017,641.1227,25.3,857.9676,216.8448,0.0832,5.4115,69.2781,417.5714,487.2024,1580.8868,19.4255,417.8407,1068.8446,0,2,61.0,488.0,289.0,42.0,199.0,3.278688524590164,200.0,10.894,19.6772,0.04918032786885246,3.0,0.03278688524590164,2.0,0.03278688524590164,2.0,,,,,,2
|
||
|
|
job_591150,0.0,0,64,512,2,1.7188,1,1.0,6598.4668,9521.9474,6088.8693,29419,1,7508.721,0.1484,0.014,0.1069,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0709,0.0,0.0,1.0,0.0,0.0109,0.2969,0.1484,12.9957,51.341,637.1793,25.7,857.9676,220.7883,0.0935,4.8247,53.3349,340.9789,394.6829,675.1796,19.7755,341.254,255.893,0,3,60.0,480.0,411.0,22.0,69.0,1.05,63.0,,,,,,,0.016666666666666666,1.0,0.016666666666666666,1.0,,,,3
|
||
|
|
job_591150,0.0,0,64,512,3,2.4219,1,1.0,9999.6543,9601.1462,10199.4927,29478,1,7901.9526,0.334,0.0083,0.0995,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1012,0.0,0.0,1.0,0.0,0.0119,0.4531,0.334,13.3472,51.7056,629.7285,26.6,857.9676,228.2391,0.0901,5.533,67.9116,401.1539,469.4069,1064.6399,19.7968,401.4046,569.8905,0,4,65.0,520.0,384.0,31.0,136.0,2.0153846153846153,131.0,0.0057,12.0885,,,,,0.07692307692307693,5.0,,,,,,4
|
||
|
|
job_591150,0.0,0,64,512,4,3.0625,2,1.0,9386.873,11132.7622,8710.2818,29967,1,8316.5867,0.2793,-0.0005,0.1511,-0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0843,-0.0,0.0,1.0,0.0,0.0126,0.4375,0.2793,13.3903,51.7529,626.9168,26.9,857.9676,231.0508,0.0708,5.2362,79.0552,433.729,513.1297,1514.2234,19.1602,434.0033,976.6598,0,5,62.0,496.0,384.0,32.0,112.0,1.9193548387096775,119.0,,,,,,,,,,,6.6576,,,5
|
||
|
|
job_591150,0.0,0,64,512,4,3.7656,3,1.0,10051.041,12819.4872,8837.9017,29419,1,8180.7526,0.3047,0.0255,0.2105,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0891,0.0,0.0,1.0,0.0,0.0202,0.5469,0.3047,13.8707,51.8625,639.0039,25.5,857.9676,218.9637,0.077,5.2138,69.1587,408.4466,477.9141,961.7158,18.673,408.6779,459.8809,0,6,63.0,504.0,316.0,42.0,187.0,2.9365079365079363,185.0,0.0083,9.9275,,,,,0.031746031746031744,2.0,,,,0.015873015873015872,1.0,6
|
||
|
|
job_591150,0.0,0,64,512,5,4.7812,4,1.0,8289.875,10832.8,7553.2594,31013,1,8640.4314,0.2246,-0.0009,0.0888,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0742,0.0,0.0,1.0,0.0,0.0102,0.3281,0.2246,13.6701,51.6724,636.2298,25.8,857.9676,221.7378,0.0697,5.0448,66.0801,402.6137,469.0069,1264.3273,18.9369,402.8567,771.3001,0,7,60.0,480.0,268.0,53.0,211.0,3.45,207.0,,,0.03333333333333333,2.0,,,0.06666666666666667,4.0,,,,0.016666666666666666,1.0,7
|
||
|
|
job_591150,0.0,0,64,512,6,5.7656,5,1.0,8230.7988,14968.2024,6908.5047,29456,1,9244.7451,0.1641,0.0138,0.1045,-0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0659,-0.0,0.0,1.0,0.0,0.0139,0.3438,0.1641,14.0063,52.0513,628.7164,26.7,857.9676,229.2512,0.065,4.8541,69.1673,417.4759,486.978,2605.5652,11.9483,417.7453,2101.7529,0,8,64.0,512.0,487.0,7.0,25.0,0.390625,25.0,0.008,10.0222,,,,,,,,,,,,8
|
||
|
|
job_591150,0.0,0,64,512,0,0.0,0,0.0,10444.8711,9636.092,11216.6069,30180,1,6434.7455,0.4883,0.0112,0.1585,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.1128,0.0,0.0,1.0,0.0,0.0149,0.6562,0.4883,13.9713,52.0759,624.9913,27.2,857.9676,232.9763,0.0687,5.276,68.7787,378.0224,447.1007,3884.0127,18.4616,378.2529,3413.142,1,9,58.0,464.0,301.0,36.0,163.0,3.0689655172413794,178.0,,,0.05172413793103448,3.0,,,,,,,,,,9
|
||
|
|
job_591150,0.0,0,64,512,1,1.0,1,1.0,8861.4961,11804.4167,7709.9185,29409,1,8814.3882,0.2812,0.0029,0.0973,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0734,0.0,0.0,1.0,0.0,0.0095,0.4062,0.2812,14.7997,52.574,627.1608,26.9,857.9676,230.8068,0.0644,5.1711,72.0844,417.3814,489.7789,926.0357,19.0802,417.6298,412.0023,1,10,64.0,512.0,332.0,36.0,179.0,2.640625,169.0,0.0076,9.5829,0.046875,3.0,0.015625,1.0,0.015625,1.0,0.03125,2.0,6.3468,,,10
|
||
|
|
job_591150,0.0,0,64,512,2,1.7031,0,0.9531,7327.1992,9194.4762,6845.4693,28487,1,7502.5285,0.2051,0.0026,0.0657,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0804,0.0,0.0,1.0,0.0,0.01,0.2812,0.2051,15.1613,52.9576,624.4334,27.2,857.9676,233.5342,0.0601,4.715,63.8687,351.7918,415.97,995.055,19.1877,352.0408,555.1789,1,11,62.0,496.0,358.0,30.0,138.0,2.193548387096774,136.0,,,0.03225806451612903,2.0,,,,,,,,,,11
|
||
|
|
job_591150,0.0,0,64,512,3,2.0938,1,1.0,9918.1191,11967.1758,8943.7839,30360,1,8420.1283,0.3223,0.0095,0.1672,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0848,0.0,0.0,1.0,0.0,0.0144,0.4844,0.3223,15.4885,53.2342,634.7371,26.0,857.9676,223.2305,0.0831,5.3606,76.0698,426.1412,502.5426,1539.2977,19.1177,426.3893,1012.2655,1,12,62.0,496.0,314.0,40.0,181.0,2.8870967741935485,179.0,0.0072,9.3904,0.06451612903225806,4.0,0.016129032258064516,1.0,,,,,,,,12
|
||
|
|
job_591150,0.0,0,64,512,4,2.5938,2,1.0,9382.0039,12392.648,8409.5736,30099,1,9042.8211,0.2441,-0.0046,0.1564,-0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0751,-0.002,0.0,1.0,0.0,0.0148,0.4219,0.2441,15.3411,53.0607,632.1736,26.3,857.9676,225.7939,0.0688,5.2259,70.3939,431.8179,502.543,1232.3428,19.0569,432.0799,705.4787,1,13,64.0,512.0,303.0,44.0,209.0,3.203125,205.0,,,0.0625,4.0,,,,,,,,,,13
|
||
|
|
job_591150,0.0,0,64,512,4,3.0469,3,1.0,8458.4961,12622.1583,7183.9056,30096,1,8927.1653,0.2344,-0.0072,0.1074,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0716,0.0,0.0,1.0,0.0,0.0087,0.375,0.2344,15.6031,53.179,628.9873,26.7,857.9676,228.9803,0.0787,5.1752,73.5778,416.5122,490.4141,1429.1938,19.0701,416.7571,914.4966,1,14,60.0,480.0,334.0,33.0,146.0,2.716666666666667,163.0,0.0072,9.1814,0.03333333333333333,2.0,,,,,,,,,,14
|
||
|
|
job_591150,0.0,0,64,512,4,4.0,4,1.0,9658.8066,12490.4551,8288.1246,29820,1,8982.0547,0.3262,0.0033,0.186,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0761,0.0,0.0,1.0,0.0,0.0122,0.4688,0.3262,15.4732,53.0505,626.9476,26.9,857.9676,231.02,0.0661,5.096,76.3654,444.7385,521.4854,1171.3137,18.8645,445.0534,625.8348,1,15,63.0,504.0,225.0,54.0,278.0,4.349206349206349,274.0,,,0.06349206349206349,4.0,,,,,,,5.8824,0.015873015873015872,1.0,15
|
||
|
|
job_591150,0.0,0,64,512,5,5.0,5,1.0,7392.418,16034.6364,6113.5247,29621,1,9341.5481,0.1289,-0.0013,0.1187,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0553,0.0,0.0,1.0,0.0,0.0095,0.2812,0.1289,15.7165,53.3323,624.8472,27.2,857.9676,233.1204,0.0635,4.8164,68.4974,418.1021,486.9176,2531.0981,13.3106,418.3564,2026.0146,1,16,,,,,,,,0.0074,9.3875,,,,,,,,,,,,16
|