dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

[muh] delay v2 完成: 全部 8 个 lookback 算法改为 no_delay

Browse Source 
基于 CCCL single_pass_scan_operators.cuh 源码分析:
  delay() 在 gridDim.x < 500 时只做 __threadfence_block，不 __nanosleep
  BI-V100: 16 SMs → max 32 CTAs → 永远 < 500

变更文件:
  tuning_reduce_by_key.cuh: 全部 66 条 → no_delay (已在上个 commit)
  tuning_scan_by_key.cuh: 全部 ~76 条 → no_delay (已在上个 commit)
  tuning_select_if.cuh: 38 个 scale_delay() → nd(l2w), 删除 scale_delay 函数
  tuning_unique_by_key.cuh: 31 个 sd() → nd(l2w), 删除 sd 函数
  tuning_three_way_partition.cuh: 6 个 sd() → nd(l2w)
  tuning_rle_encode.cuh: 5 个 sd() → nd(l2w)
  tuning_rle_non_trivial_runs.cuh: 5 个 sd() → nd(l2w)
  tuning_scan.cuh: 12 个 exponential_* → no_delay

L2WriteLatency 全部保留 (CCCL 构造函数一次性 L2 write 等待)
threads/items/load_algorithm/load_modifier 不变 (CCCL benchmark-tuned)
This commit is contained in:
muh-bot
2026-08-04 07:18:48 +00:00
parent a898faa34e
commit 11032fe95e
6 changed files with 161 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
muh/include/muh/tuning/tuning_rle_encode.cuh
View File

@@ -20,8 +20,8 @@ struct policy_selector {
bool key_is_primitive;
static constexpr LookbackDelayPolicy nd(int l2w) { return {LookbackDelayAlgorithm::no_delay, 0, l2w}; }
static constexpr LookbackDelayPolicy sd(LookbackDelayAlgorithm a, int ns, int l2w) {
return {a, (int)(ns*0.5), (int)(l2w*0.6)};
static constexpr LookbackDelayPolicy nd(int l2w) {
return {LookbackDelayAlgorithm::no_delay, 0, l2w};
}
constexpr RleLookbackPolicy p(int tpb, int ipt, BlockLoadAlgorithm la,
@@ -32,27 +32,27 @@ struct policy_selector {
constexpr RleLookbackPolicy dispatch() const {
if (!key_is_primitive) {
if (key_size==16) return p(128, 11, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
{LookbackDelayAlgorithm::fixed_delay, 428, 930});
nd(930));
int ipt = 6 * 8 / (key_size + 4); if (ipt<1) ipt=1; if (ipt>6) ipt=6;
return p(128, ipt, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, {LookbackDelayAlgorithm::fixed_delay, 350, 450});
return p(128, ipt, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, nd(450));
}
// SM100 (delay scaled)
// ipt_14.tpb_256.trp_0.ld_1.ns_468.dcid_7.l2w_300
if (key_size==1) return p(256, 14, BLOCK_LOAD_DIRECT, LOAD_CA,
sd(LookbackDelayAlgorithm::exponential_backon, 468, 300));
nd(300));
// ipt_14.tpb_224.trp_0.ld_0.ns_376.dcid_7.l2w_420
if (key_size==2) return p(224, 14, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
sd(LookbackDelayAlgorithm::exponential_backon, 376, 420));
nd(420));
// ipt_14.tpb_256.trp_0.ld_1.ns_956.dcid_7.l2w_70
if (key_size==4) return p(256, 14, BLOCK_LOAD_DIRECT, LOAD_CA,
sd(LookbackDelayAlgorithm::exponential_backon, 956, 70));
nd(70));
// ipt_9.tpb_224.trp_1.ld_0.ns_188.dcid_2.l2w_765
if (key_size==8) return p(224, 9, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
sd(LookbackDelayAlgorithm::exponential_backoff, 188, 765));
nd(765));
int ipt = 6 * 8 / (key_size + 4); if (ipt<1) ipt=1; if (ipt>6) ipt=6;
return p(128, ipt, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, {LookbackDelayAlgorithm::fixed_delay, 350, 450});
return p(128, ipt, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, nd(450));
}
constexpr RleEncodePolicy operator()(const hardware_capability& hw) const {

18
muh/include/muh/tuning/tuning_rle_non_trivial_runs.cuh
View File

@@ -25,8 +25,8 @@ struct policy_selector {
bool key_is_primitive;
static constexpr LookbackDelayPolicy nd(int l2w) { return {LookbackDelayAlgorithm::no_delay, 0, l2w}; }
static constexpr LookbackDelayPolicy sd(LookbackDelayAlgorithm a, int ns, int l2w) {
return {a, (int)(ns*0.5), (int)(l2w*0.6)};
static constexpr LookbackDelayPolicy nd(int l2w) {
return {LookbackDelayAlgorithm::no_delay, 0, l2w};
}
constexpr RleNonTrivialRunsLookbackPolicy p(int tpb, int ipt, BlockLoadAlgorithm la,
@@ -38,26 +38,26 @@ struct policy_selector {
if (!key_is_primitive) {
// int128: SM90 entry
if (key_size==16) return p(288, 9, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
{LookbackDelayAlgorithm::fixed_delay, 484, 1150});
nd(1150));
// Default
int ipt = 15 * 4 / key_size; if (ipt<1) ipt=1; if (ipt>15) ipt=15;
return p(96, ipt, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, {LookbackDelayAlgorithm::fixed_delay, 350, 450});
return p(96, ipt, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, nd(450));
}
// SM100 (delay scaled)
if (key_size==1) return p(224, 20, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_CA,
sd(LookbackDelayAlgorithm::exponential_backoff, 64, 315));
nd(315));
if (key_size==2) return p(224, 20, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
sd(LookbackDelayAlgorithm::exponential_backon, 116, 340));
nd(340));
if (key_size==4) return p(224, 13, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
sd(LookbackDelayAlgorithm::exponential_backoff, 252, 470));
nd(470));
if (key_size==8) return p(256, 15, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
sd(LookbackDelayAlgorithm::exponential_backoff, 28, 520));
nd(520));
// SM90 fallback
// (SM100 already covers 1/2/4/8, this handles edge cases)
int ipt = 15 * 4 / key_size; if (ipt<1) ipt=1; if (ipt>15) ipt=15;
return p(96, ipt, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, {LookbackDelayAlgorithm::fixed_delay, 350, 450});
return p(96, ipt, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, nd(450));
}
constexpr RleNonTrivialRunsPolicy operator()(const hardware_capability& hw) const {

14
muh/include/muh/tuning/tuning_scan.cuh
View File

@@ -95,7 +95,7 @@ struct bi100_lookback_1B_o4 {
static constexpr int threads = 512;
static constexpr int items = 32;
static constexpr LookbackDelayPolicy delay = {
LookbackDelayAlgorithm::exponential_backon, 384, 492};
LookbackDelayAlgorithm::no_delay, 0, 492};
static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE;
static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE;
static constexpr CacheLoadModifier load_mod = LOAD_DEFAULT;
@@ -108,7 +108,7 @@ struct bi100_lookback_2B_o4 {
static constexpr int threads = 512;
static constexpr int items = 24;
static constexpr LookbackDelayPolicy delay = {
LookbackDelayAlgorithm::exponential_backon, 692, 432};
LookbackDelayAlgorithm::no_delay, 0, 432};
static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE;
static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE;
static constexpr CacheLoadModifier load_mod = LOAD_DEFAULT;
@@ -138,7 +138,7 @@ struct bi100_lookback_4B_o8 {
static constexpr int threads = 416;
static constexpr int items = 19;
static constexpr LookbackDelayPolicy delay = {
LookbackDelayAlgorithm::exponential_backon, 478, 330};
LookbackDelayAlgorithm::no_delay, 0, 330};
static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE;
static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE;
static constexpr CacheLoadModifier load_mod = LOAD_CA;
@@ -151,7 +151,7 @@ struct bi100_lookback_8B_o4 {
static constexpr int threads = 384;
static constexpr int items = 16;
static constexpr LookbackDelayPolicy delay = {
LookbackDelayAlgorithm::exponential_backon_jitter_window, 386, 426};
LookbackDelayAlgorithm::no_delay, 0, 426};
static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE;
static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE;
static constexpr CacheLoadModifier load_mod = LOAD_DEFAULT;
@@ -164,7 +164,7 @@ struct bi100_lookback_8B_o8 {
static constexpr int threads = 320;
static constexpr int items = 19;
static constexpr LookbackDelayPolicy delay = {
LookbackDelayAlgorithm::exponential_backoff, 164, 579};
LookbackDelayAlgorithm::no_delay, 0, 579};
static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE;
static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE;
static constexpr CacheLoadModifier load_mod = LOAD_DEFAULT;
@@ -177,7 +177,7 @@ struct bi100_lookback_1B_o8 {
static constexpr int threads = 384;
static constexpr int items = 14;
static constexpr LookbackDelayPolicy delay = {
LookbackDelayAlgorithm::exponential_backon, 114, 465};
LookbackDelayAlgorithm::no_delay, 0, 465};
static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE;
static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE;
static constexpr CacheLoadModifier load_mod = LOAD_CA;
@@ -234,7 +234,7 @@ struct bi100_lookback_default {
static constexpr int threads = 128;
static constexpr int items = 15;
static constexpr LookbackDelayPolicy delay = {
LookbackDelayAlgorithm::fixed_delay, 350, 450};
LookbackDelayAlgorithm::no_delay, 0, 450};
static constexpr BlockLoadAlgorithm load_algo = BLOCK_LOAD_WARP_TRANSPOSE;
static constexpr BlockStoreAlgorithm store_algo = BLOCK_STORE_WARP_TRANSPOSE;
static constexpr CacheLoadModifier load_mod = LOAD_DEFAULT;

126
muh/include/muh/tuning/tuning_select_if.cuh
View File

@@ -20,10 +20,11 @@
// BLOCK_LOAD_WARP_TRANSPOSE adds: threads * items * input_size (staging buffer)
// With flags: + threads * items * 1 (bool flag per element)
//
// Delay scaling rationale:
// SM100 L2 = 50MB, BI-V100 L2 = 6MB → 8.3x smaller
// SM100 BW = 3.35 TB/s, BI-V100 BW = 900 GB/s → 3.7x slower
// Empirical: ns * 0.5, l2w * 0.6 (conservative, pending benchmark)
// DELAY v2: all no_delay (based on CCCL delay system source analysis)
// CCCL delay() checks gridDim.x < 500 → __threadfence_block() (no nanosleep)
// BI-V100: 16 SMs × ~2 CTAs/SM = max 32 CTAs → always < 500
// → exponential_backoff/backon are wasted cycles on BI-V100
// L2WriteLatency preserved (one-time constructor wait for L2 visibility)
#pragma once
@@ -108,10 +109,9 @@ struct policy_selector {
return {threads, items, load_alg, load_mod, BLOCK_SCAN_WARP_SCANS, delay};
}
// Scale SM100 delay for BI-V100
static constexpr LookbackDelayPolicy scale_delay(
LookbackDelayAlgorithm algo, int ns, int l2w) {
return {algo, static_cast<int>(ns * 0.5), static_cast<int>(l2w * 0.6)};
// BI-V100: always no_delay (16 SMs → gridDim.x < 500 → CCCL skips __nanosleep)
static constexpr LookbackDelayPolicy nd(int l2w) {
return {LookbackDelayAlgorithm::no_delay, 0, l2w};
}
// ============================================================================
@@ -127,26 +127,26 @@ struct policy_selector {
// select::if (no flags, no alias)
switch (input_size) {
case 1: return {992, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::no_delay, 0, 395}};
nd(395)};
case 2: return {576, 14, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::no_delay, 0, 870}};
nd(870)};
case 4: return {256, 18, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::no_delay, 0, 1130}};
nd(1130)};
case 8: return {192, 10, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::fixed_delay, 832, 1165}};
nd(1165)};
}
}
if (fl && !may_alias) {
// select::flagged
switch (input_size) {
case 1: return {224, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::no_delay, 0, 735}};
nd(735)};
case 2: return {256, 20, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::no_delay, 0, 1155}};
nd(1155)};
case 4: return {320, 10, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::fixed_delay, 124, 1115}};
nd(1115)};
case 8: return {384, 6, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::no_delay, 0, 1130}};
nd(1130)};
}
}
@@ -157,7 +157,7 @@ struct policy_selector {
if (items > nominal_items) items = nominal_items;
CacheLoadModifier mod = may_alias ? LOAD_CA : LOAD_LDG;
return {128, items, BLOCK_LOAD_DIRECT, mod, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::fixed_delay, 350, 450}};
nd(450)};
}
// ============================================================================
@@ -171,26 +171,26 @@ struct policy_selector {
// select::if
switch (input_size) {
case 1: return {256, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::no_delay, 0, 580}};
nd(580)};
case 2: return {256, 22, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::fixed_delay, 320, 605}};
nd(605)};
case 4: return {384, 17, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::fixed_delay, 76, 1150}};
nd(1150)};
case 8: return {384, 11, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::fixed_delay, 380, 1140}};
nd(1140)};
}
}
if (fl && !may_alias) {
// select::flagged
switch (input_size) {
case 1: return {448, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::no_delay, 0, 715}};
nd(715)};
case 2: return {448, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::fixed_delay, 504, 765}};
nd(765)};
case 4: return {384, 15, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::fixed_delay, 415, 1125}};
nd(1125)};
case 8: return {384, 11, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::fixed_delay, 360, 1170}};
nd(1170)};
}
}
@@ -211,26 +211,26 @@ struct policy_selector {
bool fl = has_flags();
constexpr SelectLookbackPolicy NO_MATCH = {0, 0, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::fixed_delay, 0, 0}};
nd(0)};
// ---- select::if (no flags, no keep_rejects) ----
if (!fl && !may_alias && offset_size == 4) {
if (input_size == 1) {
// trp_0.ld_0.ipt_22.tpb_384.ns_0.dcid_2.l2w_915
return make_safe_policy(384, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backoff, 0, 915));
nd(915));
}
if (input_size == 4) {
// trp_1.ld_0.ipt_15.tpb_384.ns_1508.dcid_5.l2w_585
return make_safe_policy(384, 15, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1508, 585));
nd(585));
}
}
if (!fl && may_alias && offset_size == 4) {
if (input_size == 1) {
// trp_1.ld_0.ipt_20.tpb_448.ns_596.dcid_6.l2w_295
return make_safe_policy(448, 20, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 596, 295));
nd(295));
}
}
@@ -240,44 +240,44 @@ struct policy_selector {
// trp_0.ld_0.ipt_20.tpb_896.ns_84.dcid_7.l2w_480
// NOTE: tpb=896 may exceed SM=16 occupancy, keep for throughput
return make_safe_policy(896, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon, 84, 480));
nd(480));
}
if (input_size == 2) {
// trp_0.ld_0.ipt_22.tpb_256.ns_1292.dcid_5.l2w_750
return make_safe_policy(256, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1292, 750));
nd(750));
}
if (input_size == 4) {
// trp_0.ld_0.ipt_14.tpb_512.ns_844.dcid_6.l2w_675
return make_safe_policy(512, 14, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 844, 675));
nd(675));
}
if (input_size == 8) {
// trp_0.ld_1.ipt_22.tpb_320.ns_660.dcid_7.l2w_1030
return make_safe_policy(320, 22, BLOCK_LOAD_DIRECT, LOAD_CA,
scale_delay(LookbackDelayAlgorithm::exponential_backon, 660, 1030));
nd(1030));
}
}
if (fl && may_alias && offset_size == 4) {
if (input_size == 1) {
// trp_0.ld_0.ipt_20.tpb_1024.ns_360.dcid_6.l2w_380
return make_safe_policy(1024, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 360, 380));
nd(380));
}
if (input_size == 2) {
// trp_1.ld_0.ipt_20.tpb_448.ns_136.dcid_2.l2w_760
return make_safe_policy(448, 20, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backoff, 136, 760));
nd(760));
}
if (input_size == 4) {
// trp_1.ld_0.ipt_14.tpb_384.ns_524.dcid_7.l2w_635
return make_safe_policy(384, 14, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon, 524, 635));
nd(635));
}
if (input_size == 8) {
// trp_1.ld_1.ipt_21.tpb_384.ns_1316.dcid_5.l2w_990
return make_safe_policy(384, 21, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_CA,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1316, 990));
nd(990));
}
}
@@ -286,32 +286,32 @@ struct policy_selector {
if (offset_size == 4 && input_size == 1) {
// trp_0.ld_0.ipt_15.tpb_608.ns_676.dcid_7.l2w_500
return make_safe_policy(608, 15, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon, 676, 500));
nd(500));
}
if (offset_size == 4 && input_size == 2) {
// trp_0.ld_0.ipt_22.tpb_320.ns_1756.dcid_6.l2w_615
return make_safe_policy(320, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 1756, 615));
nd(615));
}
if (offset_size == 4 && input_size == 4) {
// trp_1.ld_0.ipt_19.tpb_320.ns_716.dcid_5.l2w_570
return make_safe_policy(320, 19, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 716, 570));
nd(570));
}
if (offset_size == 8 && input_size == 1) {
// trp_0.ld_0.ipt_22.tpb_576.ns_368.dcid_7.l2w_680
return make_safe_policy(576, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon, 368, 680));
nd(680));
}
if (offset_size == 8 && input_size == 2) {
// trp_1.ld_0.ipt_20.tpb_608.ns_516.dcid_7.l2w_635
return make_safe_policy(608, 20, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 516, 635));
nd(635));
}
if (offset_size == 8 && input_size == 4) {
// trp_1.ld_0.ipt_18.tpb_608.ns_1712.dcid_5.l2w_825
return make_safe_policy(608, 18, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1712, 825));
nd(825));
}
}
@@ -320,37 +320,37 @@ struct policy_selector {
if (offset_size == 4 && input_size == 1) {
// trp_0.ld_0.ipt_22.tpb_224.ns_68.dcid_2.l2w_990
return make_safe_policy(224, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backoff, 68, 990));
nd(990));
}
if (offset_size == 4 && input_size == 2) {
// trp_0.ld_0.ipt_22.tpb_320.ns_560.dcid_5.l2w_640
return make_safe_policy(320, 22, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 560, 640));
nd(640));
}
if (offset_size == 4 && input_size == 4) {
// trp_1.ld_0.ipt_19.tpb_608.ns_724.dcid_5.l2w_970
return make_safe_policy(608, 19, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 724, 970));
nd(970));
}
if (offset_size == 8 && input_size == 1) {
// trp_0.ld_0.ipt_20.tpb_608.ns_1016.dcid_6.l2w_545
return make_safe_policy(608, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 1016, 545));
nd(545));
}
if (offset_size == 8 && input_size == 2) {
// trp_1.ld_0.ipt_22.tpb_288.ns_124.dcid_2.l2w_690
return make_safe_policy(288, 22, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backoff, 124, 690));
nd(690));
}
if (offset_size == 8 && input_size == 4) {
// trp_1.ld_0.ipt_19.tpb_608.ns_1884.dcid_6.l2w_950
return make_safe_policy(608, 19, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 1884, 950));
nd(950));
}
if (offset_size == 8 && input_size == 8) {
// trp_1.ld_0.ipt_23.tpb_416.ns_0.dcid_2.l2w_1200
return make_safe_policy(416, 23, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backoff, 0, 1200));
nd(1200));
}
}
@@ -359,32 +359,32 @@ struct policy_selector {
if (offset_size == 4 && input_size == 1) {
// trp_0.ld_0.ipt_20.tpb_448.ns_964.dcid_7.l2w_385
return make_safe_policy(448, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon, 964, 385));
nd(385));
}
if (offset_size == 4 && input_size == 8) {
// trp_0.ld_0.ipt_21.tpb_384.ns_300.dcid_7.l2w_580
return make_safe_policy(384, 21, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon, 300, 580));
nd(580));
}
if (offset_size == 8 && input_size == 1) {
// trp_0.ld_1.ipt_20.tpb_448.ns_240.dcid_6.l2w_845
return make_safe_policy(448, 20, BLOCK_LOAD_DIRECT, LOAD_CA,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter, 240, 845));
nd(845));
}
if (offset_size == 8 && input_size == 2) {
// trp_0.ld_0.ipt_14.tpb_320.ns_1428.dcid_7.l2w_830
return make_safe_policy(320, 14, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon, 1428, 830));
nd(830));
}
if (offset_size == 8 && input_size == 4) {
// trp_0.ld_0.ipt_14.tpb_640.ns_1204.dcid_5.l2w_635
return make_safe_policy(640, 14, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1204, 635));
nd(635));
}
if (offset_size == 8 && input_size == 8) {
// trp_0.ld_0.ipt_19.tpb_384.ns_1016.dcid_7.l2w_875
return make_safe_policy(384, 19, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon, 1016, 875));
nd(875));
}
}
@@ -393,37 +393,37 @@ struct policy_selector {
if (offset_size == 4 && input_size == 1) {
// trp_0.ld_0.ipt_24.tpb_256.ns_2024.dcid_5.l2w_835
return make_safe_policy(256, 24, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 2024, 835));
nd(835));
}
if (offset_size == 4 && input_size == 4) {
// trp_0.ld_0.ipt_11.tpb_448.ns_476.dcid_7.l2w_665
return make_safe_policy(448, 11, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon, 476, 665));
nd(665));
}
if (offset_size == 4 && input_size == 8) {
// trp_0.ld_0.ipt_20.tpb_384.ns_1420.dcid_5.l2w_525
return make_safe_policy(384, 20, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1420, 525));
nd(525));
}
if (offset_size == 8 && input_size == 1) {
// trp_0.ld_0.ipt_12.tpb_256.ns_0.dcid_5.l2w_850
return make_safe_policy(256, 12, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 0, 850));
nd(850));
}
if (offset_size == 8 && input_size == 2) {
// trp_0.ld_0.ipt_12.tpb_256.ns_1552.dcid_7.l2w_730
return make_safe_policy(256, 12, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon, 1552, 730));
nd(730));
}
if (offset_size == 8 && input_size == 4) {
// trp_0.ld_0.ipt_14.tpb_352.ns_1444.dcid_5.l2w_655
return make_safe_policy(352, 14, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backon_jitter_window, 1444, 655));
nd(655));
}
if (offset_size == 8 && input_size == 8) {
// trp_0.ld_0.ipt_11.tpb_512.ns_536.dcid_2.l2w_845
return make_safe_policy(512, 11, BLOCK_LOAD_DIRECT, LOAD_DEFAULT,
scale_delay(LookbackDelayAlgorithm::exponential_backoff, 536, 845));
nd(845));
}
}

34
muh/include/muh/tuning/tuning_three_way_partition.cuh
View File

@@ -39,54 +39,54 @@ struct policy_selector {
return {tpb, ipt, la, lm, BLOCK_SCAN_WARP_SCANS, d};
}
static constexpr LookbackDelayPolicy sd(LookbackDelayAlgorithm a, int ns, int l2w) {
return {a, (int)(ns*0.5), (int)(l2w*0.6)};
static constexpr LookbackDelayPolicy nd(int l2w) {
return {LookbackDelayAlgorithm::no_delay, 0, l2w};
}
constexpr ThreeWayPartitionLookbackPolicy default_policy() const {
int items = 9 * 4 / input_size;
if (items < 1) items = 1; if (items > 9) items = 9;
return {256, items, BLOCK_LOAD_DIRECT, LOAD_DEFAULT, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::fixed_delay, 350, 450}};
nd(450)};
}
constexpr ThreeWayPartitionLookbackPolicy dispatch() const {
// === SM100 tunings (delay scaled for BI-V100) ===
if (offset_size==4 && input_size==4)
return safe(512,11,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter,72,840));
return safe(512,11,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(840));
if (offset_size==4 && input_size==8)
return safe(256,10,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter,8,845));
return safe(256,10,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(845));
if (offset_size==8 && input_size==2)
return safe(768,20,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,544,500));
return safe(768,20,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(500));
if (offset_size==8 && input_size==4)
return safe(768,15,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter,144,280));
return safe(768,15,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(280));
if (offset_size==8 && input_size==8)
return safe(320,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,872,620));
return safe(320,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(620));
// === SM90 tunings ===
if (offset_size==4 && input_size==1)
return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,445});
return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(445));
if (offset_size==4 && input_size==2)
return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,104,512});
return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(512));
// SM90 offset=4 input=4 already covered by SM100 above
// SM90 offset=4 input=8 already covered
if (offset_size==4 && input_size==16)
return safe(128,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1040});
return safe(128,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1040));
if (offset_size==8 && input_size==1)
return safe(256,24,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,4,285});
return safe(256,24,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(285));
// offset=8 input=2,4,8 covered by SM100
if (offset_size==8 && input_size==16)
return safe(256,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1050});
return safe(256,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1050));
// === SM80 tunings ===
if (offset_size==4 && input_size==2)
return safe(256,12,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,910});
return safe(256,12,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(910));
if (offset_size==4 && input_size==4)
return safe(256,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1120});
return safe(256,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1120));
if (offset_size==4 && input_size==8)
return safe(224,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,264,1080});
return safe(224,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1080));
if (offset_size==4 && input_size==16)
return safe(128,10,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,672,1120});
return safe(128,10,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1120));
return default_policy();
}

112
muh/include/muh/tuning/tuning_unique_by_key.cuh
View File

@@ -49,7 +49,7 @@ struct policy_selector {
return {tpb, ipt, la, lm, BLOCK_SCAN_WARP_SCANS, d};
}
static constexpr LookbackDelayPolicy sd(LookbackDelayAlgorithm a, int ns, int l2w) {
static constexpr LookbackDelayPolicy nd(int l2w) {
return {a, (int)(ns * 0.5), (int)(l2w * 0.6)};
}
@@ -57,96 +57,96 @@ struct policy_selector {
int items = 11 * 4 / key_size;
if (items < 1) items = 1; if (items > 11) items = 11;
return {64, items, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_LDG, BLOCK_SCAN_WARP_SCANS,
{LookbackDelayAlgorithm::fixed_delay, 350, 450}};
nd(450)};
}
// SM100 tuning — 15 benchmark entries, delay scaled for BI-V100
constexpr UniqueByKeyPolicy get_sm100() const {
constexpr UniqueByKeyPolicy NONE = {0,0,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,BLOCK_SCAN_WARP_SCANS,{LookbackDelayAlgorithm::fixed_delay,0,0}};
constexpr UniqueByKeyPolicy NONE = {0,0,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,BLOCK_SCAN_WARP_SCANS,nd(0)};
if (!primitive_key) return NONE;
if (!primitive_value) return NONE;
// key=1B
if (key_size==1 && value_size==1) return safe(512,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,948,955));
if (key_size==1 && value_size==2) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,1228,320));
if (key_size==1 && value_size==4) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,2016,620));
if (key_size==1 && value_size==8) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,1728,980));
if (key_size==1 && value_size==1) return safe(512,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(955));
if (key_size==1 && value_size==2) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(320));
if (key_size==1 && value_size==4) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(620));
if (key_size==1 && value_size==8) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(980));
// key=2B
if (key_size==2 && value_size==1) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,508,1020));
if (key_size==2 && value_size==2) return safe(384,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,928,605));
if (key_size==2 && value_size==4) return safe(384,11,BLOCK_LOAD_DIRECT,LOAD_CA,sd(LookbackDelayAlgorithm::exponential_backon,1620,810));
if (key_size==2 && value_size==8) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,1984,935));
if (key_size==2 && value_size==1) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1020));
if (key_size==2 && value_size==2) return safe(384,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(605));
if (key_size==2 && value_size==4) return safe(384,11,BLOCK_LOAD_DIRECT,LOAD_CA,nd(810));
if (key_size==2 && value_size==8) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(935));
// key=4B
if (key_size==4 && value_size==1) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,1136,605));
if (key_size==4 && value_size==2) return safe(384,11,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon,656,825));
if (key_size==4 && value_size==8) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,1012,800));
if (key_size==4 && value_size==1) return safe(512,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(605));
if (key_size==4 && value_size==2) return safe(384,11,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(825));
if (key_size==4 && value_size==8) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(800));
// key=8B
if (key_size==8 && value_size==2) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,864,1130));
if (key_size==8 && value_size==4) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,sd(LookbackDelayAlgorithm::exponential_backon_jitter_window,772,665));
if (key_size==8 && value_size==2) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1130));
if (key_size==8 && value_size==4) return safe(384,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(665));
return NONE;
}
// SM90 tuning — 24 entries (20 primitive + 4 val_size=16)
constexpr UniqueByKeyPolicy get_sm90() const {
constexpr UniqueByKeyPolicy NONE = {0,0,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,BLOCK_SCAN_WARP_SCANS,{LookbackDelayAlgorithm::fixed_delay,0,0}};
constexpr UniqueByKeyPolicy NONE = {0,0,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,BLOCK_SCAN_WARP_SCANS,nd(0)};
if (!primitive_key) return NONE;
if (primitive_value) {
if (key_size==1 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,550});
if (key_size==1 && value_size==2) return safe(448,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,725});
if (key_size==1 && value_size==4) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1130});
if (key_size==1 && value_size==8) return safe(512,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1100});
if (key_size==2 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,640});
if (key_size==2 && value_size==2) return safe(288,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,404,710});
if (key_size==2 && value_size==4) return safe(512,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,525});
if (key_size==2 && value_size==8) return safe(256,23,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1200});
if (key_size==4 && value_size==1) return safe(448,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,348,580});
if (key_size==4 && value_size==2) return safe(384,9,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1060});
if (key_size==4 && value_size==4) return safe(512,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1045});
if (key_size==4 && value_size==8) return safe(512,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1120});
if (key_size==8 && value_size==1) return safe(384,9,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1060});
if (key_size==8 && value_size==2) return safe(384,9,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,964,1125});
if (key_size==8 && value_size==4) return safe(640,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1070});
if (key_size==8 && value_size==8) return safe(448,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1190});
if (key_size==1 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(550));
if (key_size==1 && value_size==2) return safe(448,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(725));
if (key_size==1 && value_size==4) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1130));
if (key_size==1 && value_size==8) return safe(512,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1100));
if (key_size==2 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(640));
if (key_size==2 && value_size==2) return safe(288,14,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(710));
if (key_size==2 && value_size==4) return safe(512,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(525));
if (key_size==2 && value_size==8) return safe(256,23,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1200));
if (key_size==4 && value_size==1) return safe(448,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(580));
if (key_size==4 && value_size==2) return safe(384,9,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1060));
if (key_size==4 && value_size==4) return safe(512,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1045));
if (key_size==4 && value_size==8) return safe(512,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1120));
if (key_size==8 && value_size==1) return safe(384,9,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1060));
if (key_size==8 && value_size==2) return safe(384,9,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1125));
if (key_size==8 && value_size==4) return safe(640,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1070));
if (key_size==8 && value_size==8) return safe(448,11,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1190));
}
// non-primitive value, size=16
if (value_size == 16) {
if (key_size==1) return safe(288,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,344,1165});
if (key_size==2) return safe(224,9,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,424,1055});
if (key_size==4) return safe(384,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1025});
if (key_size==8) return safe(256,9,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1155});
if (key_size==1) return safe(288,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1165));
if (key_size==2) return safe(224,9,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1055));
if (key_size==4) return safe(384,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1025));
if (key_size==8) return safe(256,9,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1155));
}
return NONE;
}
// SM80 tuning — 32 entries
constexpr UniqueByKeyPolicy get_sm80() const {
constexpr UniqueByKeyPolicy NONE = {0,0,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,BLOCK_SCAN_WARP_SCANS,{LookbackDelayAlgorithm::fixed_delay,0,0}};
constexpr UniqueByKeyPolicy NONE = {0,0,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,BLOCK_SCAN_WARP_SCANS,nd(0)};
if (!primitive_key) return NONE;
if (primitive_value) {
if (key_size==1 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,835});
if (key_size==1 && value_size==2) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,765});
if (key_size==1 && value_size==4) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1155});
if (key_size==1 && value_size==8) return safe(224,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1065});
if (key_size==2 && value_size==1) return safe(320,20,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1020});
if (key_size==2 && value_size==2) return safe(192,22,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,328,1080});
if (key_size==2 && value_size==4) return safe(256,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,535});
if (key_size==2 && value_size==8) return safe(256,10,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1055});
if (key_size==4 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1120});
if (key_size==4 && value_size==2) return safe(256,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1185});
if (key_size==4 && value_size==4) return safe(256,11,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::no_delay,0,1115});
if (key_size==4 && value_size==8) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,320,1115});
if (key_size==8 && value_size==1) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,24,555});
if (key_size==8 && value_size==2) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,324,1105});
if (key_size==8 && value_size==4) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,740,1105});
if (key_size==8 && value_size==8) return safe(192,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,764,1155});
if (key_size==1 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(835));
if (key_size==1 && value_size==2) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(765));
if (key_size==1 && value_size==4) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1155));
if (key_size==1 && value_size==8) return safe(224,10,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1065));
if (key_size==2 && value_size==1) return safe(320,20,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1020));
if (key_size==2 && value_size==2) return safe(192,22,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1080));
if (key_size==2 && value_size==4) return safe(256,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(535));
if (key_size==2 && value_size==8) return safe(256,10,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1055));
if (key_size==4 && value_size==1) return safe(256,12,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1120));
if (key_size==4 && value_size==2) return safe(256,14,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1185));
if (key_size==4 && value_size==4) return safe(256,11,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1115));
if (key_size==4 && value_size==8) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1115));
if (key_size==8 && value_size==1) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(555));
if (key_size==8 && value_size==2) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1105));
if (key_size==8 && value_size==4) return safe(256,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1105));
if (key_size==8 && value_size==8) return safe(192,7,BLOCK_LOAD_DIRECT,LOAD_DEFAULT,nd(1155));
}
// non-primitive val, size=16
if (value_size == 16) {
if (key_size==1) return safe(128,15,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,248,1200});
if (key_size==8) return safe(128,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,{LookbackDelayAlgorithm::fixed_delay,992,1135});
if (key_size==1) return safe(128,15,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1200));
if (key_size==8) return safe(128,7,BLOCK_LOAD_WARP_TRANSPOSE,LOAD_DEFAULT,nd(1135));
}
return NONE;
}