diff --git a/muh/include/muh/tuning/tuning_rle_encode.cuh b/muh/include/muh/tuning/tuning_rle_encode.cuh index ea35a9f6..671cb01d 100644 --- a/muh/include/muh/tuning/tuning_rle_encode.cuh +++ b/muh/include/muh/tuning/tuning_rle_encode.cuh @@ -20,8 +20,8 @@ struct policy_selector { bool key_is_primitive; static constexpr LookbackDelayPolicy nd(int l2w) { return {LookbackDelayAlgorithm::no_delay, 0, l2w}; } - static constexpr LookbackDelayPolicy sd(LookbackDelayAlgorithm a, int ns, int l2w) { - return {a, (int)(ns*0.5), (int)(l2w*0.6)}; + static constexpr LookbackDelayPolicy nd(int l2w) { + return {LookbackDelayAlgorithm::no_delay, 0, l2w}; } constexpr RleLookbackPolicy p(int tpb, int ipt, BlockLoadAlgorithm la, @@ -32,27 +32,27 @@ struct policy_selector { constexpr RleLookbackPolicy dispatch() const { if (!key_is_primitive) { if (key_size==16) return p(128, 11, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - {LookbackDelayAlgorithm::fixed_delay, 428, 930}); + nd(930)); int ipt = 6 * 8 / (key_size + 4); if (ipt<1) ipt=1; if (ipt>6) ipt=6; - return p(128, ipt, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, {LookbackDelayAlgorithm::fixed_delay, 350, 450}); + return p(128, ipt, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, nd(450)); } // SM100 (delay scaled) // ipt_14.tpb_256.trp_0.ld_1.ns_468.dcid_7.l2w_300 if (key_size==1) return p(256, 14, BLOCK_LOAD_DIRECT, LOAD_CA, - sd(LookbackDelayAlgorithm::exponential_backon, 468, 300)); + nd(300)); // ipt_14.tpb_224.trp_0.ld_0.ns_376.dcid_7.l2w_420 if (key_size==2) return p(224, 14, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - sd(LookbackDelayAlgorithm::exponential_backon, 376, 420)); + nd(420)); // ipt_14.tpb_256.trp_0.ld_1.ns_956.dcid_7.l2w_70 if (key_size==4) return p(256, 14, BLOCK_LOAD_DIRECT, LOAD_CA, - sd(LookbackDelayAlgorithm::exponential_backon, 956, 70)); + nd(70)); // ipt_9.tpb_224.trp_1.ld_0.ns_188.dcid_2.l2w_765 if (key_size==8) return p(224, 9, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - sd(LookbackDelayAlgorithm::exponential_backoff, 188, 765)); + nd(765)); int ipt = 6 * 8 / (key_size + 4); if (ipt<1) ipt=1; if (ipt>6) ipt=6; - return p(128, ipt, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, {LookbackDelayAlgorithm::fixed_delay, 350, 450}); + return p(128, ipt, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, nd(450)); } constexpr RleEncodePolicy operator()(const hardware_capability& hw) const { diff --git a/muh/include/muh/tuning/tuning_rle_non_trivial_runs.cuh b/muh/include/muh/tuning/tuning_rle_non_trivial_runs.cuh index 010ee5e8..93b9bd79 100644 --- a/muh/include/muh/tuning/tuning_rle_non_trivial_runs.cuh +++ b/muh/include/muh/tuning/tuning_rle_non_trivial_runs.cuh @@ -25,8 +25,8 @@ struct policy_selector { bool key_is_primitive; static constexpr LookbackDelayPolicy nd(int l2w) { return {LookbackDelayAlgorithm::no_delay, 0, l2w}; } - static constexpr LookbackDelayPolicy sd(LookbackDelayAlgorithm a, int ns, int l2w) { - return {a, (int)(ns*0.5), (int)(l2w*0.6)}; + static constexpr LookbackDelayPolicy nd(int l2w) { + return {LookbackDelayAlgorithm::no_delay, 0, l2w}; } constexpr RleNonTrivialRunsLookbackPolicy p(int tpb, int ipt, BlockLoadAlgorithm la, @@ -38,26 +38,26 @@ struct policy_selector { if (!key_is_primitive) { // int128: SM90 entry if (key_size==16) return p(288, 9, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - {LookbackDelayAlgorithm::fixed_delay, 484, 1150}); + nd(1150)); // Default int ipt = 15 * 4 / key_size; if (ipt<1) ipt=1; if (ipt>15) ipt=15; - return p(96, ipt, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, {LookbackDelayAlgorithm::fixed_delay, 350, 450}); + return p(96, ipt, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, nd(450)); } // SM100 (delay scaled) if (key_size==1) return p(224, 20, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_CA, - sd(LookbackDelayAlgorithm::exponential_backoff, 64, 315)); + nd(315)); if (key_size==2) return p(224, 20, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - sd(LookbackDelayAlgorithm::exponential_backon, 116, 340)); + nd(340)); if (key_size==4) return p(224, 13, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - sd(LookbackDelayAlgorithm::exponential_backoff, 252, 470)); + nd(470)); if (key_size==8) return p(256, 15, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - sd(LookbackDelayAlgorithm::exponential_backoff, 28, 520)); + nd(520)); // SM90 fallback // (SM100 already covers 1/2/4/8, this handles edge cases) int ipt = 15 * 4 / key_size; if (ipt<1) ipt=1; if (ipt>15) ipt=15; - return p(96, ipt, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, {LookbackDelayAlgorithm::fixed_delay, 350, 450}); + return p(96, ipt, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, nd(450)); } constexpr RleNonTrivialRunsPolicy operator()(const hardware_capability& hw) const { diff --git a/muh/include/muh/tuning/tuning_scan.cuh b/muh/include/muh/tuning/tuning_scan.cuh index efdf1cbf..1d3db041 100644 --- a/muh/include/muh/tuning/tuning_scan.cuh +++ b/muh/include/muh/tuning/tuning_scan.cuh @@ -95,7 +95,7 @@ struct bi100_lookback_1B_o4 { static constexpr int threads = 512; static constexpr int items = 32; static constexpr LookbackDelayPolicy delay = { - LookbackDelayAlgorithm::exponential_backon, 384, 492}; + LookbackDelayAlgorithm::no_delay, 0, 492}; static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE; static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE; static constexpr CacheLoadModifier load_mod = LOAD_DEFAULT; @@ -108,7 +108,7 @@ struct bi100_lookback_2B_o4 { static constexpr int threads = 512; static constexpr int items = 24; static constexpr LookbackDelayPolicy delay = { - LookbackDelayAlgorithm::exponential_backon, 692, 432}; + LookbackDelayAlgorithm::no_delay, 0, 432}; static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE; static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE; static constexpr CacheLoadModifier load_mod = LOAD_DEFAULT; @@ -138,7 +138,7 @@ struct bi100_lookback_4B_o8 { static constexpr int threads = 416; static constexpr int items = 19; static constexpr LookbackDelayPolicy delay = { - LookbackDelayAlgorithm::exponential_backon, 478, 330}; + LookbackDelayAlgorithm::no_delay, 0, 330}; static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE; static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE; static constexpr CacheLoadModifier load_mod = LOAD_CA; @@ -151,7 +151,7 @@ struct bi100_lookback_8B_o4 { static constexpr int threads = 384; static constexpr int items = 16; static constexpr LookbackDelayPolicy delay = { - LookbackDelayAlgorithm::exponential_backon_jitter_window, 386, 426}; + LookbackDelayAlgorithm::no_delay, 0, 426}; static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE; static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE; static constexpr CacheLoadModifier load_mod = LOAD_DEFAULT; @@ -164,7 +164,7 @@ struct bi100_lookback_8B_o8 { static constexpr int threads = 320; static constexpr int items = 19; static constexpr LookbackDelayPolicy delay = { - LookbackDelayAlgorithm::exponential_backoff, 164, 579}; + LookbackDelayAlgorithm::no_delay, 0, 579}; static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE; static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE; static constexpr CacheLoadModifier load_mod = LOAD_DEFAULT; @@ -177,7 +177,7 @@ struct bi100_lookback_1B_o8 { static constexpr int threads = 384; static constexpr int items = 14; static constexpr LookbackDelayPolicy delay = { - LookbackDelayAlgorithm::exponential_backon, 114, 465}; + LookbackDelayAlgorithm::no_delay, 0, 465}; static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE; static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE; static constexpr CacheLoadModifier load_mod = LOAD_CA; @@ -234,7 +234,7 @@ struct bi100_lookback_default { static constexpr int threads = 128; static constexpr int items = 15; static constexpr LookbackDelayPolicy delay = { - LookbackDelayAlgorithm::fixed_delay, 350, 450}; + LookbackDelayAlgorithm::no_delay, 0, 450}; static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE; static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE; static constexpr CacheLoadModifier load_mod = LOAD_DEFAULT; diff --git a/muh/include/muh/tuning/tuning_select_if.cuh b/muh/include/muh/tuning/tuning_select_if.cuh index b1c2c8a9..a4d2ed6a 100644 --- a/muh/include/muh/tuning/tuning_select_if.cuh +++ b/muh/include/muh/tuning/tuning_select_if.cuh @@ -20,10 +20,11 @@ // BLOCK_LOAD_WARP_TRANSPOSE adds: threads * items * input_size (staging buffer) // With flags: + threads * items * 1 (bool flag per element) // -// Delay scaling rationale: -// SM100 L2 = 50MB, BI-V100 L2 = 6MB → 8.3x smaller -// SM100 BW = 3.35 TB/s, BI-V100 BW = 900 GB/s → 3.7x slower -// Empirical: ns * 0.5, l2w * 0.6 (conservative, pending benchmark) +// DELAY v2: all no_delay (based on CCCL delay system source analysis) +// CCCL delay() checks gridDim.x < 500 → __threadfence_block() (no nanosleep) +// BI-V100: 16 SMs × ~2 CTAs/SM = max 32 CTAs → always < 500 +// → exponential_backoff/backon are wasted cycles on BI-V100 +// L2WriteLatency preserved (one-time constructor wait for L2 visibility) #pragma once @@ -108,10 +109,9 @@ struct policy_selector { return {threads, items, load_alg, load_mod, BLOCK_SCAN_WARP_SCANS, delay}; } - // Scale SM100 delay for BI-V100 - static constexpr LookbackDelayPolicy scale_delay( - LookbackDelayAlgorithm algo, int ns, int l2w) { - return {algo, static_cast(ns * 0.5), static_cast(l2w * 0.6)}; + // BI-V100: always no_delay (16 SMs → gridDim.x < 500 → CCCL skips __nanosleep) + static constexpr LookbackDelayPolicy nd(int l2w) { + return {LookbackDelayAlgorithm::no_delay, 0, l2w}; } // ============================================================================ @@ -127,26 +127,26 @@ struct policy_selector { // select::if (no flags, no alias) switch (input_size) { case 1: return {992, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::no_delay, 0, 395}}; + nd(395)}; case 2: return {576, 14, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::no_delay, 0, 870}}; + nd(870)}; case 4: return {256, 18, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::no_delay, 0, 1130}}; + nd(1130)}; case 8: return {192, 10, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::fixed_delay, 832, 1165}}; + nd(1165)}; } } if (fl && !may_alias) { // select::flagged switch (input_size) { case 1: return {224, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::no_delay, 0, 735}}; + nd(735)}; case 2: return {256, 20, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::no_delay, 0, 1155}}; + nd(1155)}; case 4: return {320, 10, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::fixed_delay, 124, 1115}}; + nd(1115)}; case 8: return {384, 6, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::no_delay, 0, 1130}}; + nd(1130)}; } } @@ -157,7 +157,7 @@ struct policy_selector { if (items > nominal_items) items = nominal_items; CacheLoadModifier mod = may_alias ? LOAD_CA : LOAD_LDG; return {128, items, BLOCK_LOAD_DIRECT, mod, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::fixed_delay, 350, 450}}; + nd(450)}; } // ============================================================================ @@ -171,26 +171,26 @@ struct policy_selector { // select::if switch (input_size) { case 1: return {256, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::no_delay, 0, 580}}; + nd(580)}; case 2: return {256, 22, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::fixed_delay, 320, 605}}; + nd(605)}; case 4: return {384, 17, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::fixed_delay, 76, 1150}}; + nd(1150)}; case 8: return {384, 11, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::fixed_delay, 380, 1140}}; + nd(1140)}; } } if (fl && !may_alias) { // select::flagged switch (input_size) { case 1: return {448, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::no_delay, 0, 715}}; + nd(715)}; case 2: return {448, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::fixed_delay, 504, 765}}; + nd(765)}; case 4: return {384, 15, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::fixed_delay, 415, 1125}}; + nd(1125)}; case 8: return {384, 11, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::fixed_delay, 360, 1170}}; + nd(1170)}; } } @@ -211,26 +211,26 @@ struct policy_selector { bool fl = has_flags(); constexpr SelectLookbackPolicy NO_MATCH = {0, 0, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::fixed_delay, 0, 0}}; + nd(0)}; // ---- select::if (no flags, no keep_rejects) ---- if (!fl && !may_alias && offset_size == 4) { if (input_size == 1) { // trp_0.ld_0.ipt_22.tpb_384.ns_0.dcid_2.l2w_915 return make_safe_policy(384, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backoff, 0, 915)); + nd(915)); } if (input_size == 4) { // trp_1.ld_0.ipt_15.tpb_384.ns_1508.dcid_5.l2w_585 return make_safe_policy(384, 15, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1508, 585)); + nd(585)); } } if (!fl && may_alias && offset_size == 4) { if (input_size == 1) { // trp_1.ld_0.ipt_20.tpb_448.ns_596.dcid_6.l2w_295 return make_safe_policy(448, 20, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 596, 295)); + nd(295)); } } @@ -240,44 +240,44 @@ struct policy_selector { // trp_0.ld_0.ipt_20.tpb_896.ns_84.dcid_7.l2w_480 // NOTE: tpb=896 may exceed SM=16 occupancy, keep for throughput return make_safe_policy(896, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon, 84, 480)); + nd(480)); } if (input_size == 2) { // trp_0.ld_0.ipt_22.tpb_256.ns_1292.dcid_5.l2w_750 return make_safe_policy(256, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1292, 750)); + nd(750)); } if (input_size == 4) { // trp_0.ld_0.ipt_14.tpb_512.ns_844.dcid_6.l2w_675 return make_safe_policy(512, 14, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 844, 675)); + nd(675)); } if (input_size == 8) { // trp_0.ld_1.ipt_22.tpb_320.ns_660.dcid_7.l2w_1030 return make_safe_policy(320, 22, BLOCK_LOAD_DIRECT, LOAD_CA, - scale_delay(LookbackDelayAlgorithm::exponential_backon, 660, 1030)); + nd(1030)); } } if (fl && may_alias && offset_size == 4) { if (input_size == 1) { // trp_0.ld_0.ipt_20.tpb_1024.ns_360.dcid_6.l2w_380 return make_safe_policy(1024, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 360, 380)); + nd(380)); } if (input_size == 2) { // trp_1.ld_0.ipt_20.tpb_448.ns_136.dcid_2.l2w_760 return make_safe_policy(448, 20, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backoff, 136, 760)); + nd(760)); } if (input_size == 4) { // trp_1.ld_0.ipt_14.tpb_384.ns_524.dcid_7.l2w_635 return make_safe_policy(384, 14, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon, 524, 635)); + nd(635)); } if (input_size == 8) { // trp_1.ld_1.ipt_21.tpb_384.ns_1316.dcid_5.l2w_990 return make_safe_policy(384, 21, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_CA, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1316, 990)); + nd(990)); } } @@ -286,32 +286,32 @@ struct policy_selector { if (offset_size == 4 && input_size == 1) { // trp_0.ld_0.ipt_15.tpb_608.ns_676.dcid_7.l2w_500 return make_safe_policy(608, 15, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon, 676, 500)); + nd(500)); } if (offset_size == 4 && input_size == 2) { // trp_0.ld_0.ipt_22.tpb_320.ns_1756.dcid_6.l2w_615 return make_safe_policy(320, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 1756, 615)); + nd(615)); } if (offset_size == 4 && input_size == 4) { // trp_1.ld_0.ipt_19.tpb_320.ns_716.dcid_5.l2w_570 return make_safe_policy(320, 19, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 716, 570)); + nd(570)); } if (offset_size == 8 && input_size == 1) { // trp_0.ld_0.ipt_22.tpb_576.ns_368.dcid_7.l2w_680 return make_safe_policy(576, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon, 368, 680)); + nd(680)); } if (offset_size == 8 && input_size == 2) { // trp_1.ld_0.ipt_20.tpb_608.ns_516.dcid_7.l2w_635 return make_safe_policy(608, 20, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 516, 635)); + nd(635)); } if (offset_size == 8 && input_size == 4) { // trp_1.ld_0.ipt_18.tpb_608.ns_1712.dcid_5.l2w_825 return make_safe_policy(608, 18, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1712, 825)); + nd(825)); } } @@ -320,37 +320,37 @@ struct policy_selector { if (offset_size == 4 && input_size == 1) { // trp_0.ld_0.ipt_22.tpb_224.ns_68.dcid_2.l2w_990 return make_safe_policy(224, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backoff, 68, 990)); + nd(990)); } if (offset_size == 4 && input_size == 2) { // trp_0.ld_0.ipt_22.tpb_320.ns_560.dcid_5.l2w_640 return make_safe_policy(320, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 560, 640)); + nd(640)); } if (offset_size == 4 && input_size == 4) { // trp_1.ld_0.ipt_19.tpb_608.ns_724.dcid_5.l2w_970 return make_safe_policy(608, 19, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 724, 970)); + nd(970)); } if (offset_size == 8 && input_size == 1) { // trp_0.ld_0.ipt_20.tpb_608.ns_1016.dcid_6.l2w_545 return make_safe_policy(608, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 1016, 545)); + nd(545)); } if (offset_size == 8 && input_size == 2) { // trp_1.ld_0.ipt_22.tpb_288.ns_124.dcid_2.l2w_690 return make_safe_policy(288, 22, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backoff, 124, 690)); + nd(690)); } if (offset_size == 8 && input_size == 4) { // trp_1.ld_0.ipt_19.tpb_608.ns_1884.dcid_6.l2w_950 return make_safe_policy(608, 19, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 1884, 950)); + nd(950)); } if (offset_size == 8 && input_size == 8) { // trp_1.ld_0.ipt_23.tpb_416.ns_0.dcid_2.l2w_1200 return make_safe_policy(416, 23, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backoff, 0, 1200)); + nd(1200)); } } @@ -359,32 +359,32 @@ struct policy_selector { if (offset_size == 4 && input_size == 1) { // trp_0.ld_0.ipt_20.tpb_448.ns_964.dcid_7.l2w_385 return make_safe_policy(448, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon, 964, 385)); + nd(385)); } if (offset_size == 4 && input_size == 8) { // trp_0.ld_0.ipt_21.tpb_384.ns_300.dcid_7.l2w_580 return make_safe_policy(384, 21, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon, 300, 580)); + nd(580)); } if (offset_size == 8 && input_size == 1) { // trp_0.ld_1.ipt_20.tpb_448.ns_240.dcid_6.l2w_845 return make_safe_policy(448, 20, BLOCK_LOAD_DIRECT, LOAD_CA, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 240, 845)); + nd(845)); } if (offset_size == 8 && input_size == 2) { // trp_0.ld_0.ipt_14.tpb_320.ns_1428.dcid_7.l2w_830 return make_safe_policy(320, 14, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon, 1428, 830)); + nd(830)); } if (offset_size == 8 && input_size == 4) { // trp_0.ld_0.ipt_14.tpb_640.ns_1204.dcid_5.l2w_635 return make_safe_policy(640, 14, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1204, 635)); + nd(635)); } if (offset_size == 8 && input_size == 8) { // trp_0.ld_0.ipt_19.tpb_384.ns_1016.dcid_7.l2w_875 return make_safe_policy(384, 19, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon, 1016, 875)); + nd(875)); } } @@ -393,37 +393,37 @@ struct policy_selector { if (offset_size == 4 && input_size == 1) { // trp_0.ld_0.ipt_24.tpb_256.ns_2024.dcid_5.l2w_835 return make_safe_policy(256, 24, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 2024, 835)); + nd(835)); } if (offset_size == 4 && input_size == 4) { // trp_0.ld_0.ipt_11.tpb_448.ns_476.dcid_7.l2w_665 return make_safe_policy(448, 11, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon, 476, 665)); + nd(665)); } if (offset_size == 4 && input_size == 8) { // trp_0.ld_0.ipt_20.tpb_384.ns_1420.dcid_5.l2w_525 return make_safe_policy(384, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1420, 525)); + nd(525)); } if (offset_size == 8 && input_size == 1) { // trp_0.ld_0.ipt_12.tpb_256.ns_0.dcid_5.l2w_850 return make_safe_policy(256, 12, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 0, 850)); + nd(850)); } if (offset_size == 8 && input_size == 2) { // trp_0.ld_0.ipt_12.tpb_256.ns_1552.dcid_7.l2w_730 return make_safe_policy(256, 12, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon, 1552, 730)); + nd(730)); } if (offset_size == 8 && input_size == 4) { // trp_0.ld_0.ipt_14.tpb_352.ns_1444.dcid_5.l2w_655 return make_safe_policy(352, 14, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1444, 655)); + nd(655)); } if (offset_size == 8 && input_size == 8) { // trp_0.ld_0.ipt_11.tpb_512.ns_536.dcid_2.l2w_845 return make_safe_policy(512, 11, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, - scale_delay(LookbackDelayAlgorithm::exponential_backoff, 536, 845)); + nd(845)); } } diff --git a/muh/include/muh/tuning/tuning_three_way_partition.cuh b/muh/include/muh/tuning/tuning_three_way_partition.cuh index 893d9aee..26e25b78 100644 --- a/muh/include/muh/tuning/tuning_three_way_partition.cuh +++ b/muh/include/muh/tuning/tuning_three_way_partition.cuh @@ -39,54 +39,54 @@ struct policy_selector { return {tpb, ipt, la, lm, BLOCK_SCAN_WARP_SCANS, d}; } - static constexpr LookbackDelayPolicy sd(LookbackDelayAlgorithm a, int ns, int l2w) { - return {a, (int)(ns*0.5), (int)(l2w*0.6)}; + static constexpr LookbackDelayPolicy nd(int l2w) { + return {LookbackDelayAlgorithm::no_delay, 0, l2w}; } constexpr ThreeWayPartitionLookbackPolicy default_policy() const { int items = 9 * 4 / input_size; if (items < 1) items = 1; if (items > 9) items = 9; return {256, items, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::fixed_delay, 350, 450}}; + nd(450)}; } constexpr ThreeWayPartitionLookbackPolicy dispatch() const { // === SM100 tunings (delay scaled for BI-V100) === if (offset_size==4 && input_size==4) - return safe(512,11,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter,72,840)); + return safe(512,11,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(840)); if (offset_size==4 && input_size==8) - return safe(256,10,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter,8,845)); + return safe(256,10,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(845)); if (offset_size==8 && input_size==2) - return safe(768,20,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,544,500)); + return safe(768,20,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(500)); if (offset_size==8 && input_size==4) - return safe(768,15,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter,144,280)); + return safe(768,15,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(280)); if (offset_size==8 && input_size==8) - return safe(320,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,872,620)); + return safe(320,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(620)); // === SM90 tunings === if (offset_size==4 && input_size==1) - return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,445}); + return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(445)); if (offset_size==4 && input_size==2) - return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,104,512}); + return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(512)); // SM90 offset=4 input=4 already covered by SM100 above // SM90 offset=4 input=8 already covered if (offset_size==4 && input_size==16) - return safe(128,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1040}); + return safe(128,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1040)); if (offset_size==8 && input_size==1) - return safe(256,24,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,4,285}); + return safe(256,24,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(285)); // offset=8 input=2,4,8 covered by SM100 if (offset_size==8 && input_size==16) - return safe(256,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1050}); + return safe(256,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1050)); // === SM80 tunings === if (offset_size==4 && input_size==2) - return safe(256,12,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,910}); + return safe(256,12,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(910)); if (offset_size==4 && input_size==4) - return safe(256,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1120}); + return safe(256,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1120)); if (offset_size==4 && input_size==8) - return safe(224,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,264,1080}); + return safe(224,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1080)); if (offset_size==4 && input_size==16) - return safe(128,10,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,672,1120}); + return safe(128,10,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1120)); return default_policy(); } diff --git a/muh/include/muh/tuning/tuning_unique_by_key.cuh b/muh/include/muh/tuning/tuning_unique_by_key.cuh index f0be40ea..1e340487 100644 --- a/muh/include/muh/tuning/tuning_unique_by_key.cuh +++ b/muh/include/muh/tuning/tuning_unique_by_key.cuh @@ -49,7 +49,7 @@ struct policy_selector { return {tpb, ipt, la, lm, BLOCK_SCAN_WARP_SCANS, d}; } - static constexpr LookbackDelayPolicy sd(LookbackDelayAlgorithm a, int ns, int l2w) { + static constexpr LookbackDelayPolicy nd(int l2w) { return {a, (int)(ns * 0.5), (int)(l2w * 0.6)}; } @@ -57,96 +57,96 @@ struct policy_selector { int items = 11 * 4 / key_size; if (items < 1) items = 1; if (items > 11) items = 11; return {64, items, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_LDG, BLOCK_SCAN_WARP_SCANS, - {LookbackDelayAlgorithm::fixed_delay, 350, 450}}; + nd(450)}; } // SM100 tuning — 15 benchmark entries, delay scaled for BI-V100 constexpr UniqueByKeyPolicy get_sm100() const { - constexpr UniqueByKeyPolicy NONE = {0,0,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,BLOCK_SCAN_WARP_SCANS,{LookbackDelayAlgorithm::fixed_delay,0,0}}; + constexpr UniqueByKeyPolicy NONE = {0,0,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,BLOCK_SCAN_WARP_SCANS,nd(0)}; if (!primitive_key) return NONE; if (!primitive_value) return NONE; // key=1B - if (key_size==1 && value_size==1) return safe(512,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,948,955)); - if (key_size==1 && value_size==2) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,1228,320)); - if (key_size==1 && value_size==4) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,2016,620)); - if (key_size==1 && value_size==8) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,1728,980)); + if (key_size==1 && value_size==1) return safe(512,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(955)); + if (key_size==1 && value_size==2) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(320)); + if (key_size==1 && value_size==4) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(620)); + if (key_size==1 && value_size==8) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(980)); // key=2B - if (key_size==2 && value_size==1) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,508,1020)); - if (key_size==2 && value_size==2) return safe(384,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,928,605)); - if (key_size==2 && value_size==4) return safe(384,11,BLOCK_LOAD_DIRECT,LOAD_CA,sd(LookbackDelayAlgorithm::exponential_backon,1620,810)); - if (key_size==2 && value_size==8) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,1984,935)); + if (key_size==2 && value_size==1) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1020)); + if (key_size==2 && value_size==2) return safe(384,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(605)); + if (key_size==2 && value_size==4) return safe(384,11,BLOCK_LOAD_DIRECT,LOAD_CA,nd(810)); + if (key_size==2 && value_size==8) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(935)); // key=4B - if (key_size==4 && value_size==1) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,1136,605)); - if (key_size==4 && value_size==2) return safe(384,11,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,656,825)); - if (key_size==4 && value_size==8) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,1012,800)); + if (key_size==4 && value_size==1) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(605)); + if (key_size==4 && value_size==2) return safe(384,11,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(825)); + if (key_size==4 && value_size==8) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(800)); // key=8B - if (key_size==8 && value_size==2) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,864,1130)); - if (key_size==8 && value_size==4) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,772,665)); + if (key_size==8 && value_size==2) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1130)); + if (key_size==8 && value_size==4) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(665)); return NONE; } // SM90 tuning — 24 entries (20 primitive + 4 val_size=16) constexpr UniqueByKeyPolicy get_sm90() const { - constexpr UniqueByKeyPolicy NONE = {0,0,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,BLOCK_SCAN_WARP_SCANS,{LookbackDelayAlgorithm::fixed_delay,0,0}}; + constexpr UniqueByKeyPolicy NONE = {0,0,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,BLOCK_SCAN_WARP_SCANS,nd(0)}; if (!primitive_key) return NONE; if (primitive_value) { - if (key_size==1 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,550}); - if (key_size==1 && value_size==2) return safe(448,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,725}); - if (key_size==1 && value_size==4) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1130}); - if (key_size==1 && value_size==8) return safe(512,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1100}); - if (key_size==2 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,640}); - if (key_size==2 && value_size==2) return safe(288,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,404,710}); - if (key_size==2 && value_size==4) return safe(512,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,525}); - if (key_size==2 && value_size==8) return safe(256,23,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1200}); - if (key_size==4 && value_size==1) return safe(448,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,348,580}); - if (key_size==4 && value_size==2) return safe(384,9,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1060}); - if (key_size==4 && value_size==4) return safe(512,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1045}); - if (key_size==4 && value_size==8) return safe(512,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1120}); - if (key_size==8 && value_size==1) return safe(384,9,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1060}); - if (key_size==8 && value_size==2) return safe(384,9,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,964,1125}); - if (key_size==8 && value_size==4) return safe(640,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1070}); - if (key_size==8 && value_size==8) return safe(448,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1190}); + if (key_size==1 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(550)); + if (key_size==1 && value_size==2) return safe(448,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(725)); + if (key_size==1 && value_size==4) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1130)); + if (key_size==1 && value_size==8) return safe(512,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1100)); + if (key_size==2 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(640)); + if (key_size==2 && value_size==2) return safe(288,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(710)); + if (key_size==2 && value_size==4) return safe(512,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(525)); + if (key_size==2 && value_size==8) return safe(256,23,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1200)); + if (key_size==4 && value_size==1) return safe(448,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(580)); + if (key_size==4 && value_size==2) return safe(384,9,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1060)); + if (key_size==4 && value_size==4) return safe(512,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1045)); + if (key_size==4 && value_size==8) return safe(512,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1120)); + if (key_size==8 && value_size==1) return safe(384,9,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1060)); + if (key_size==8 && value_size==2) return safe(384,9,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1125)); + if (key_size==8 && value_size==4) return safe(640,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1070)); + if (key_size==8 && value_size==8) return safe(448,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1190)); } // non-primitive value, size=16 if (value_size == 16) { - if (key_size==1) return safe(288,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,344,1165}); - if (key_size==2) return safe(224,9,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,424,1055}); - if (key_size==4) return safe(384,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1025}); - if (key_size==8) return safe(256,9,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1155}); + if (key_size==1) return safe(288,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1165)); + if (key_size==2) return safe(224,9,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1055)); + if (key_size==4) return safe(384,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1025)); + if (key_size==8) return safe(256,9,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1155)); } return NONE; } // SM80 tuning — 32 entries constexpr UniqueByKeyPolicy get_sm80() const { - constexpr UniqueByKeyPolicy NONE = {0,0,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,BLOCK_SCAN_WARP_SCANS,{LookbackDelayAlgorithm::fixed_delay,0,0}}; + constexpr UniqueByKeyPolicy NONE = {0,0,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,BLOCK_SCAN_WARP_SCANS,nd(0)}; if (!primitive_key) return NONE; if (primitive_value) { - if (key_size==1 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,835}); - if (key_size==1 && value_size==2) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,765}); - if (key_size==1 && value_size==4) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1155}); - if (key_size==1 && value_size==8) return safe(224,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1065}); - if (key_size==2 && value_size==1) return safe(320,20,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1020}); - if (key_size==2 && value_size==2) return safe(192,22,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,328,1080}); - if (key_size==2 && value_size==4) return safe(256,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,535}); - if (key_size==2 && value_size==8) return safe(256,10,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1055}); - if (key_size==4 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1120}); - if (key_size==4 && value_size==2) return safe(256,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1185}); - if (key_size==4 && value_size==4) return safe(256,11,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1115}); - if (key_size==4 && value_size==8) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,320,1115}); - if (key_size==8 && value_size==1) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,24,555}); - if (key_size==8 && value_size==2) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,324,1105}); - if (key_size==8 && value_size==4) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,740,1105}); - if (key_size==8 && value_size==8) return safe(192,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,764,1155}); + if (key_size==1 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(835)); + if (key_size==1 && value_size==2) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(765)); + if (key_size==1 && value_size==4) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1155)); + if (key_size==1 && value_size==8) return safe(224,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1065)); + if (key_size==2 && value_size==1) return safe(320,20,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1020)); + if (key_size==2 && value_size==2) return safe(192,22,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1080)); + if (key_size==2 && value_size==4) return safe(256,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(535)); + if (key_size==2 && value_size==8) return safe(256,10,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1055)); + if (key_size==4 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1120)); + if (key_size==4 && value_size==2) return safe(256,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1185)); + if (key_size==4 && value_size==4) return safe(256,11,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1115)); + if (key_size==4 && value_size==8) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1115)); + if (key_size==8 && value_size==1) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(555)); + if (key_size==8 && value_size==2) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1105)); + if (key_size==8 && value_size==4) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1105)); + if (key_size==8 && value_size==8) return safe(192,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1155)); } // non-primitive val, size=16 if (value_size == 16) { - if (key_size==1) return safe(128,15,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,248,1200}); - if (key_size==8) return safe(128,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,992,1135}); + if (key_size==1) return safe(128,15,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1200)); + if (key_size==8) return safe(128,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1135)); } return NONE; }