diff --git a/muh/include/muh/muh.cuh b/muh/include/muh/muh.cuh index a330a38a..0c07747b 100644 --- a/muh/include/muh/muh.cuh +++ b/muh/include/muh/muh.cuh @@ -1,28 +1,7 @@ // muh/include/muh/muh.cuh — Top-level muh header // -// Provides the complete tuning dispatch for Iluvatar BI-V100. +// Complete BI-V100 tuning dispatch for all 26 CCCL algorithms. // Include this single header to get all tuning policies. -// -// Usage: -// #include -// -// auto hw = muh::target_hw; // BI-V100 by default -// auto reduce_policy = muh::tuning::reduce::policy_selector{ -// .accum_t = muh::tuning::type_t::float32, -// .operation_t = muh::tuning::op_kind_t::plus, -// .offset_size = 4, -// .accum_size = 4, -// }(hw); -// -// auto scan_policy = muh::tuning::scan::policy_selector{ -// .input_value_size = 4, -// .accum_size = 4, -// .offset_size = 4, -// .input_type = muh::tuning::type_t::float32, -// .accum_type = muh::tuning::type_t::float32, -// .operation_t = muh::tuning::op_kind_t::plus, -// .is_primitive_accum = true, -// }(hw); #pragma once @@ -32,34 +11,52 @@ // Shared types (compatible with CCCL) #include "muh/tuning/common.cuh" -// Per-algorithm tuning (P0 = highest priority for competition) -#include "muh/tuning/tuning_reduce.cuh" // P0: attention reduction -#include "muh/tuning/tuning_topk.cuh" // P0: sampling top-k/top-p -#include "muh/tuning/tuning_scan.cuh" // P0: prefix scan in paged attention +// P0: Highest priority for competition (Output TPS × 16.796) +#include "muh/tuning/tuning_reduce.cuh" +#include "muh/tuning/tuning_topk.cuh" +#include "muh/tuning/tuning_scan.cuh" -// P1 -#include "muh/tuning/tuning_transform.cuh" // P1: activation kernels -#include "muh/tuning/tuning_batch_memcpy.cuh" // P1: KV cache management +// P1: High priority +#include "muh/tuning/tuning_transform.cuh" +#include "muh/tuning/tuning_transform_tile.cuh" +#include "muh/tuning/tuning_batch_memcpy.cuh" +#include "muh/tuning/tuning_radix_sort.cuh" +#include "muh/tuning/tuning_reduce_by_key.cuh" +#include "muh/tuning/tuning_scan_by_key.cuh" +#include "muh/tuning/tuning_select_if.cuh" +#include "muh/tuning/tuning_histogram.cuh" +#include "muh/tuning/tuning_merge.cuh" +#include "muh/tuning/tuning_merge_sort.cuh" +#include "muh/tuning/tuning_unique_by_key.cuh" +#include "muh/tuning/tuning_batched_topk.cuh" -// P2 -#include "muh/tuning/tuning_for.cuh" // P2: RoPE position encoding +// P2: Segmented/specialized +#include "muh/tuning/tuning_for.cuh" +#include "muh/tuning/tuning_segmented_reduce.cuh" +#include "muh/tuning/tuning_segmented_scan.cuh" +#include "muh/tuning/tuning_segmented_sort.cuh" +#include "muh/tuning/tuning_segmented_radix_sort.cuh" +#include "muh/tuning/tuning_three_way_partition.cuh" +#include "muh/tuning/tuning_rle_encode.cuh" +#include "muh/tuning/tuning_rle_non_trivial_runs.cuh" + +// P3: Utility +#include "muh/tuning/tuning_adjacent_difference.cuh" +#include "muh/tuning/tuning_find.cuh" +#include "muh/tuning/tuning_find_bound_sorted_values.cuh" namespace muh { - -/// Version info constexpr int MUH_VERSION_MAJOR = 0; -constexpr int MUH_VERSION_MINOR = 1; +constexpr int MUH_VERSION_MINOR = 2; constexpr int MUH_VERSION_PATCH = 0; +constexpr int MUH_ALGORITHM_COUNT = 26; -/// Competition scoring formula -/// Token吞吐加权值 = Output TPS × 16.796 + Input TPS × 2.799 + Cache TPS × 0.56 struct scoring { static constexpr double output_weight = 16.796; static constexpr double input_weight = 2.799; static constexpr double cache_weight = 0.56; - static constexpr double baseline_threshold = 8000.0; // minimum to pass - static constexpr double advanced_uplift = 0.30; // 30% for advanced prize - static constexpr double special_uplift = 0.50; // 50% for special prize + static constexpr double baseline_threshold = 8000.0; + static constexpr double advanced_uplift = 0.30; + static constexpr double special_uplift = 0.50; }; - } // namespace muh diff --git a/muh/include/muh/tuning/tuning_adjacent_difference.cuh b/muh/include/muh/tuning/tuning_adjacent_difference.cuh new file mode 100644 index 00000000..76b3943c --- /dev/null +++ b/muh/include/muh/tuning/tuning_adjacent_difference.cuh @@ -0,0 +1,40 @@ +// muh/include/muh/tuning/tuning_adjacent_difference.cuh — BI-V100 adjacent_difference tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_adjacent_difference.cuh +// vllm impact: difference calculation (minor) +// Competition weight: minimal + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::adjacent_difference { + +struct AdjacentDifferencePolicy { + int threads_per_block; + int items_per_thread; + BlockLoadAlgorithm load_algorithm; + CacheLoadModifier load_modifier; +}; + +struct bi100_default { + static constexpr int threads = 128; + static constexpr int items = 7; + static constexpr int load_algo = BLOCK_LOAD_WARP_TRANSPOSE; + static constexpr int load_mod = LOAD_LDG; +}; + +struct policy_selector { + int value_size; + + constexpr AdjacentDifferencePolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_LDG}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_LDG}; + } +}; + +} // namespace muh::tuning::adjacent_difference diff --git a/muh/include/muh/tuning/tuning_batched_topk.cuh b/muh/include/muh/tuning/tuning_batched_topk.cuh new file mode 100644 index 00000000..17318a96 --- /dev/null +++ b/muh/include/muh/tuning/tuning_batched_topk.cuh @@ -0,0 +1,38 @@ +// muh/include/muh/tuning/tuning_batched_topk.cuh — BI-V100 batched_topk tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_batched_topk.cuh +// vllm impact: batched top-k across sequences +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::batched_topk { + +struct BatchedTopkPolicy { + int threads_per_block; + int items_per_thread; + BlockLoadAlgorithm load_algorithm; +}; + +struct bi100_default { + static constexpr int threads = 256; + static constexpr int items = 16; + static constexpr int load_algo = BLOCK_LOAD_WARP_TRANSPOSE; +}; + +struct policy_selector { + int key_size; + + constexpr BatchedTopkPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_WARP_TRANSPOSE}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_WARP_TRANSPOSE}; + } +}; + +} // namespace muh::tuning::batched_topk diff --git a/muh/include/muh/tuning/tuning_find.cuh b/muh/include/muh/tuning/tuning_find.cuh new file mode 100644 index 00000000..6b918034 --- /dev/null +++ b/muh/include/muh/tuning/tuning_find.cuh @@ -0,0 +1,40 @@ +// muh/include/muh/tuning/tuning_find.cuh — BI-V100 find tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_find.cuh +// vllm impact: element search (minor) +// Competition weight: minimal + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::find { + +struct FindPolicy { + int threads_per_block; + int items_per_thread; + int vec_size; + CacheLoadModifier load_modifier; +}; + +struct bi100_default { + static constexpr int threads = 128; + static constexpr int items = 16; + static constexpr int vec_size = 4; + static constexpr int load_mod = LOAD_LDG; +}; + +struct policy_selector { + int input_size; + + constexpr FindPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, bi100_default::vec_size, LOAD_LDG}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, bi100_default::vec_size, LOAD_LDG}; + } +}; + +} // namespace muh::tuning::find diff --git a/muh/include/muh/tuning/tuning_find_bound_sorted_values.cuh b/muh/include/muh/tuning/tuning_find_bound_sorted_values.cuh new file mode 100644 index 00000000..c290ec42 --- /dev/null +++ b/muh/include/muh/tuning/tuning_find_bound_sorted_values.cuh @@ -0,0 +1,39 @@ +// muh/include/muh/tuning/tuning_find_bound_sorted_values.cuh — BI-V100 find_bound_sorted_values tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_find_bound_sorted_values.cuh +// vllm impact: binary search (minor) +// Competition weight: minimal + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::find_bound_sorted_values { + +struct FindBoundSortedValuesPolicy { + int threads_per_block; + int items_per_thread; + CacheLoadModifier load_modifier; +}; + +struct bi100_default { + static constexpr int threads = 512; + static constexpr int items = 15; + static constexpr int load_mod = LOAD_DEFAULT; +}; + +struct policy_selector { + int range_size; + int values_size; + + constexpr FindBoundSortedValuesPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, LOAD_DEFAULT}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, LOAD_DEFAULT}; + } +}; + +} // namespace muh::tuning::find_bound_sorted_values diff --git a/muh/include/muh/tuning/tuning_histogram.cuh b/muh/include/muh/tuning/tuning_histogram.cuh new file mode 100644 index 00000000..d5e0c704 --- /dev/null +++ b/muh/include/muh/tuning/tuning_histogram.cuh @@ -0,0 +1,43 @@ +// muh/include/muh/tuning/tuning_histogram.cuh — BI-V100 histogram tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_histogram.cuh +// vllm impact: token frequency counting in sampling +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::histogram { + +struct HistogramPolicy { + int threads_per_block; + int pixels_per_thread; + int vec_size; + BlockLoadAlgorithm load_algorithm; + CacheLoadModifier load_modifier; +}; + +struct bi100_default { + static constexpr int threads = 768; + static constexpr int items = 12; + static constexpr int vec_size = 4; + static constexpr int load_algo = BLOCK_LOAD_DIRECT; + static constexpr int load_mod = LOAD_LDG; +}; + +struct policy_selector { + int sample_size; + int num_channels; + + constexpr HistogramPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, bi100_default::vec_size, BLOCK_LOAD_DIRECT, LOAD_LDG}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, bi100_default::vec_size, BLOCK_LOAD_DIRECT, LOAD_LDG}; + } +}; + +} // namespace muh::tuning::histogram diff --git a/muh/include/muh/tuning/tuning_merge.cuh b/muh/include/muh/tuning/tuning_merge.cuh new file mode 100644 index 00000000..bbfa4f60 --- /dev/null +++ b/muh/include/muh/tuning/tuning_merge.cuh @@ -0,0 +1,39 @@ +// muh/include/muh/tuning/tuning_merge.cuh — BI-V100 merge tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_merge.cuh +// vllm impact: sequence merge in batch scheduling +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::merge { + +struct MergePolicy { + int threads_per_block; + int items_per_thread; + CacheLoadModifier load_modifier; +}; + +struct bi100_default { + static constexpr int threads = 512; + static constexpr int items = 15; + static constexpr int load_mod = LOAD_DEFAULT; +}; + +struct policy_selector { + int key_size; + int value_size; + + constexpr MergePolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, LOAD_DEFAULT}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, LOAD_DEFAULT}; + } +}; + +} // namespace muh::tuning::merge diff --git a/muh/include/muh/tuning/tuning_merge_sort.cuh b/muh/include/muh/tuning/tuning_merge_sort.cuh new file mode 100644 index 00000000..cb3c8c7e --- /dev/null +++ b/muh/include/muh/tuning/tuning_merge_sort.cuh @@ -0,0 +1,40 @@ +// muh/include/muh/tuning/tuning_merge_sort.cuh — BI-V100 merge_sort tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_merge_sort.cuh +// vllm impact: sorting in scheduler/sampler +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::merge_sort { + +struct MergeSortPolicy { + int threads_per_block; + int items_per_thread; + BlockLoadAlgorithm load_algorithm; + CacheLoadModifier load_modifier; +}; + +struct bi100_default { + static constexpr int threads = 256; + static constexpr int items = 17; + static constexpr int load_algo = BLOCK_LOAD_WARP_TRANSPOSE; + static constexpr int load_mod = LOAD_DEFAULT; +}; + +struct policy_selector { + int key_size; + + constexpr MergeSortPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_WARP_TRANSPOSE, LOAD_DEFAULT}; + } +}; + +} // namespace muh::tuning::merge_sort diff --git a/muh/include/muh/tuning/tuning_radix_sort.cuh b/muh/include/muh/tuning/tuning_radix_sort.cuh new file mode 100644 index 00000000..80444827 --- /dev/null +++ b/muh/include/muh/tuning/tuning_radix_sort.cuh @@ -0,0 +1,40 @@ +// muh/include/muh/tuning/tuning_radix_sort.cuh — BI-V100 radix_sort tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_radix_sort.cuh +// vllm impact: beam search token ranking, scheduler sorting +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::radix_sort { + +struct RadixSortPolicy { + int threads_per_block; + int items_per_thread; + int radix_bits; +}; + +struct bi100_default { + static constexpr int threads = 384; + static constexpr int items = 23; + static constexpr int radix_bits = 8; +}; + +struct policy_selector { + int key_size; + int value_size; + int offset_size; + + constexpr RadixSortPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, bi100_default::radix_bits}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, bi100_default::radix_bits}; + } +}; + +} // namespace muh::tuning::radix_sort diff --git a/muh/include/muh/tuning/tuning_reduce_by_key.cuh b/muh/include/muh/tuning/tuning_reduce_by_key.cuh new file mode 100644 index 00000000..70286ce2 --- /dev/null +++ b/muh/include/muh/tuning/tuning_reduce_by_key.cuh @@ -0,0 +1,43 @@ +// muh/include/muh/tuning/tuning_reduce_by_key.cuh — BI-V100 reduce_by_key tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_reduce_by_key.cuh +// vllm impact: grouped reduction in multi-head attention (reduce per head) +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::reduce_by_key { + +struct ReduceByKeyPolicy { + int threads_per_block; + int items_per_thread; + BlockLoadAlgorithm load_algorithm; + CacheLoadModifier load_modifier; + LookbackDelayPolicy lookback_delay; +}; + +struct bi100_default { + static constexpr int threads = 256; + static constexpr int items = 13; + static constexpr int load_algo = BLOCK_LOAD_DIRECT; + static constexpr int load_mod = LOAD_LDG; +}; + +struct policy_selector { + int key_size; + int accum_size; + int offset_size; + + constexpr ReduceByKeyPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_DIRECT, LOAD_LDG, {LookbackDelayAlgorithm::fixed_delay, 350, 450}}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_DIRECT, LOAD_LDG, {LookbackDelayAlgorithm::fixed_delay, 350, 450}}; + } +}; + +} // namespace muh::tuning::reduce_by_key diff --git a/muh/include/muh/tuning/tuning_rle_encode.cuh b/muh/include/muh/tuning/tuning_rle_encode.cuh new file mode 100644 index 00000000..33351417 --- /dev/null +++ b/muh/include/muh/tuning/tuning_rle_encode.cuh @@ -0,0 +1,38 @@ +// muh/include/muh/tuning/tuning_rle_encode.cuh — BI-V100 rle_encode tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_rle_encode.cuh +// vllm impact: run-length encoding in sparse attention +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::rle_encode { + +struct RleEncodePolicy { + int threads_per_block; + int items_per_thread; + BlockLoadAlgorithm load_algorithm; +}; + +struct bi100_default { + static constexpr int threads = 256; + static constexpr int items = 14; + static constexpr int load_algo = BLOCK_LOAD_DIRECT; +}; + +struct policy_selector { + int key_size; + + constexpr RleEncodePolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_DIRECT}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_DIRECT}; + } +}; + +} // namespace muh::tuning::rle_encode diff --git a/muh/include/muh/tuning/tuning_rle_non_trivial_runs.cuh b/muh/include/muh/tuning/tuning_rle_non_trivial_runs.cuh new file mode 100644 index 00000000..f09e69a6 --- /dev/null +++ b/muh/include/muh/tuning/tuning_rle_non_trivial_runs.cuh @@ -0,0 +1,39 @@ +// muh/include/muh/tuning/tuning_rle_non_trivial_runs.cuh — BI-V100 rle_non_trivial_runs tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_rle_non_trivial_runs.cuh +// vllm impact: non-trivial run detection +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::rle_non_trivial_runs { + +struct RleNonTrivialRunsPolicy { + int threads_per_block; + int items_per_thread; + BlockLoadAlgorithm load_algorithm; + LookbackDelayPolicy lookback_delay; +}; + +struct bi100_default { + static constexpr int threads = 192; + static constexpr int items = 20; + static constexpr int load_algo = BLOCK_LOAD_DIRECT; +}; + +struct policy_selector { + int key_size; + + constexpr RleNonTrivialRunsPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_DIRECT, {LookbackDelayAlgorithm::fixed_delay, 350, 450}}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_DIRECT, {LookbackDelayAlgorithm::fixed_delay, 350, 450}}; + } +}; + +} // namespace muh::tuning::rle_non_trivial_runs diff --git a/muh/include/muh/tuning/tuning_scan_by_key.cuh b/muh/include/muh/tuning/tuning_scan_by_key.cuh new file mode 100644 index 00000000..0e60d7f3 --- /dev/null +++ b/muh/include/muh/tuning/tuning_scan_by_key.cuh @@ -0,0 +1,42 @@ +// muh/include/muh/tuning/tuning_scan_by_key.cuh — BI-V100 scan_by_key tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_scan_by_key.cuh +// vllm impact: attention mask prefix scan per sequence +// Competition weight: Input TPS × 2.799 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::scan_by_key { + +struct ScanByKeyPolicy { + int threads_per_block; + int items_per_thread; + BlockLoadAlgorithm load_algorithm; + BlockStoreAlgorithm store_algorithm; + LookbackDelayPolicy lookback_delay; +}; + +struct bi100_default { + static constexpr int threads = 256; + static constexpr int items = 15; + static constexpr int load_algo = BLOCK_LOAD_WARP_TRANSPOSE; + static constexpr int store_algo = BLOCK_STORE_WARP_TRANSPOSE; +}; + +struct policy_selector { + int key_size; + int accum_size; + + constexpr ScanByKeyPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_WARP_TRANSPOSE, BLOCK_STORE_WARP_TRANSPOSE, {LookbackDelayAlgorithm::fixed_delay, 350, 450}}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_WARP_TRANSPOSE, BLOCK_STORE_WARP_TRANSPOSE, {LookbackDelayAlgorithm::fixed_delay, 350, 450}}; + } +}; + +} // namespace muh::tuning::scan_by_key diff --git a/muh/include/muh/tuning/tuning_segmented_radix_sort.cuh b/muh/include/muh/tuning/tuning_segmented_radix_sort.cuh new file mode 100644 index 00000000..a588bda0 --- /dev/null +++ b/muh/include/muh/tuning/tuning_segmented_radix_sort.cuh @@ -0,0 +1,38 @@ +// muh/include/muh/tuning/tuning_segmented_radix_sort.cuh — BI-V100 segmented_radix_sort tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_segmented_radix_sort.cuh +// vllm impact: radix sort per segment +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::segmented_radix_sort { + +struct SegmentedRadixSortPolicy { + int threads_per_block; + int items_per_thread; + int radix_bits; +}; + +struct bi100_default { + static constexpr int threads = 256; + static constexpr int items = 15; + static constexpr int radix_bits = 6; +}; + +struct policy_selector { + int key_size; + + constexpr SegmentedRadixSortPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, bi100_default::radix_bits}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, bi100_default::radix_bits}; + } +}; + +} // namespace muh::tuning::segmented_radix_sort diff --git a/muh/include/muh/tuning/tuning_segmented_reduce.cuh b/muh/include/muh/tuning/tuning_segmented_reduce.cuh new file mode 100644 index 00000000..962221e1 --- /dev/null +++ b/muh/include/muh/tuning/tuning_segmented_reduce.cuh @@ -0,0 +1,39 @@ +// muh/include/muh/tuning/tuning_segmented_reduce.cuh — BI-V100 segmented_reduce tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_segmented_reduce.cuh +// vllm impact: per-segment reduce in batched attention +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::segmented_reduce { + +struct SegmentedReducePolicy { + int threads_per_block; + int items_per_thread; + int vec_size; +}; + +struct bi100_default { + static constexpr int threads = 256; + static constexpr int items = 16; + static constexpr int vec_size = 4; +}; + +struct policy_selector { + int accum_size; + int offset_size; + + constexpr SegmentedReducePolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, bi100_default::vec_size}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, bi100_default::vec_size}; + } +}; + +} // namespace muh::tuning::segmented_reduce diff --git a/muh/include/muh/tuning/tuning_segmented_scan.cuh b/muh/include/muh/tuning/tuning_segmented_scan.cuh new file mode 100644 index 00000000..edf9e20f --- /dev/null +++ b/muh/include/muh/tuning/tuning_segmented_scan.cuh @@ -0,0 +1,36 @@ +// muh/include/muh/tuning/tuning_segmented_scan.cuh — BI-V100 segmented_scan tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_segmented_scan.cuh +// vllm impact: per-segment prefix scan +// Competition weight: Input TPS × 2.799 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::segmented_scan { + +struct SegmentedScanPolicy { + int threads_per_block; + int items_per_thread; +}; + +struct bi100_default { + static constexpr int threads = 128; + static constexpr int items = 9; +}; + +struct policy_selector { + int accum_size; + + constexpr SegmentedScanPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items}; + } + // Fallback + return {bi100_default::threads, bi100_default::items}; + } +}; + +} // namespace muh::tuning::segmented_scan diff --git a/muh/include/muh/tuning/tuning_segmented_sort.cuh b/muh/include/muh/tuning/tuning_segmented_sort.cuh new file mode 100644 index 00000000..ea94bd0a --- /dev/null +++ b/muh/include/muh/tuning/tuning_segmented_sort.cuh @@ -0,0 +1,39 @@ +// muh/include/muh/tuning/tuning_segmented_sort.cuh — BI-V100 segmented_sort tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_segmented_sort.cuh +// vllm impact: per-segment sorting +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::segmented_sort { + +struct SegmentedSortPolicy { + int threads_per_block; + int items_per_thread; + int radix_bits; +}; + +struct bi100_default { + static constexpr int threads = 256; + static constexpr int items = 11; + static constexpr int radix_bits = 6; +}; + +struct policy_selector { + int key_size; + int value_size; + + constexpr SegmentedSortPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, bi100_default::radix_bits}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, bi100_default::radix_bits}; + } +}; + +} // namespace muh::tuning::segmented_sort diff --git a/muh/include/muh/tuning/tuning_select_if.cuh b/muh/include/muh/tuning/tuning_select_if.cuh new file mode 100644 index 00000000..f343a0b0 --- /dev/null +++ b/muh/include/muh/tuning/tuning_select_if.cuh @@ -0,0 +1,39 @@ +// muh/include/muh/tuning/tuning_select_if.cuh — BI-V100 select_if tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_select_if.cuh +// vllm impact: token filtering in speculative decoding +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::select_if { + +struct SelectIfPolicy { + int threads_per_block; + int items_per_thread; + BlockLoadAlgorithm load_algorithm; + LookbackDelayPolicy lookback_delay; +}; + +struct bi100_default { + static constexpr int threads = 256; + static constexpr int items = 18; + static constexpr int load_algo = BLOCK_LOAD_WARP_TRANSPOSE; +}; + +struct policy_selector { + int input_size; + + constexpr SelectIfPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_WARP_TRANSPOSE, {LookbackDelayAlgorithm::fixed_delay, 350, 450}}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_WARP_TRANSPOSE, {LookbackDelayAlgorithm::fixed_delay, 350, 450}}; + } +}; + +} // namespace muh::tuning::select_if diff --git a/muh/include/muh/tuning/tuning_three_way_partition.cuh b/muh/include/muh/tuning/tuning_three_way_partition.cuh new file mode 100644 index 00000000..d7474b9b --- /dev/null +++ b/muh/include/muh/tuning/tuning_three_way_partition.cuh @@ -0,0 +1,38 @@ +// muh/include/muh/tuning/tuning_three_way_partition.cuh — BI-V100 three_way_partition tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_three_way_partition.cuh +// vllm impact: three-way split in scheduler +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::three_way_partition { + +struct ThreeWayPartitionPolicy { + int threads_per_block; + int items_per_thread; + BlockLoadAlgorithm load_algorithm; +}; + +struct bi100_default { + static constexpr int threads = 256; + static constexpr int items = 12; + static constexpr int load_algo = BLOCK_LOAD_WARP_TRANSPOSE; +}; + +struct policy_selector { + int input_size; + + constexpr ThreeWayPartitionPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_WARP_TRANSPOSE}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_WARP_TRANSPOSE}; + } +}; + +} // namespace muh::tuning::three_way_partition diff --git a/muh/include/muh/tuning/tuning_transform_tile.cuh b/muh/include/muh/tuning/tuning_transform_tile.cuh new file mode 100644 index 00000000..ad2c148f --- /dev/null +++ b/muh/include/muh/tuning/tuning_transform_tile.cuh @@ -0,0 +1,36 @@ +// muh/include/muh/tuning/tuning_transform_tile.cuh — BI-V100 transform_tile tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_transform_tile.cuh +// vllm impact: tiled activation kernels (SiLU/GELU on tiles) +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::transform_tile { + +struct TransformTilePolicy { + int threads_per_block; + int items_per_thread; +}; + +struct bi100_default { + static constexpr int threads = 128; + static constexpr int items = 8; +}; + +struct policy_selector { + int min_elem_size; + + constexpr TransformTilePolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items}; + } + // Fallback + return {bi100_default::threads, bi100_default::items}; + } +}; + +} // namespace muh::tuning::transform_tile diff --git a/muh/include/muh/tuning/tuning_unique_by_key.cuh b/muh/include/muh/tuning/tuning_unique_by_key.cuh new file mode 100644 index 00000000..3c37cfd7 --- /dev/null +++ b/muh/include/muh/tuning/tuning_unique_by_key.cuh @@ -0,0 +1,41 @@ +// muh/include/muh/tuning/tuning_unique_by_key.cuh — BI-V100 unique_by_key tuning +// +// Mirrors: cccl_upstream/cub/cub/device/dispatch/tuning/tuning_unique_by_key.cuh +// vllm impact: deduplication in beam search +// Competition weight: Output TPS × 16.796 + +#pragma once + +#include "muh/hardware.cuh" +#include "muh/tuning/common.cuh" + +namespace muh::tuning::unique_by_key { + +struct UniqueByKeyPolicy { + int threads_per_block; + int items_per_thread; + BlockLoadAlgorithm load_algorithm; + CacheLoadModifier load_modifier; +}; + +struct bi100_default { + static constexpr int threads = 256; + static constexpr int items = 12; + static constexpr int load_algo = BLOCK_LOAD_DIRECT; + static constexpr int load_mod = LOAD_DEFAULT; +}; + +struct policy_selector { + int key_size; + int value_size; + + constexpr UniqueByKeyPolicy operator()(const hardware_capability& hw) const { + if (hw.at_least(hardware_capability::vendor_t::iluvatar, 100)) { + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_DIRECT, LOAD_DEFAULT}; + } + // Fallback + return {bi100_default::threads, bi100_default::items, BLOCK_LOAD_DIRECT, LOAD_DEFAULT}; + } +}; + +} // namespace muh::tuning::unique_by_key