// SPDX-FileCopyrightText: Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. // SPDX-License-Identifier: BSD-3 #include #include #include template __global__ void block_scan_kernel(T* in, T* out, ActionT action) { using block_scan_t = cub::BlockScan; using storage_t = typename block_scan_t::TempStorage; __shared__ storage_t storage; T thread_data[ItemsPerThread]; const int tid = static_cast(cub::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)); const int thread_offset = tid * ItemsPerThread; for (int item = 0; item < ItemsPerThread; item++) { const int idx = thread_offset + item; thread_data[item] = in[idx]; } __syncthreads(); block_scan_t scan(storage); action(scan, thread_data); for (int item = 0; item < ItemsPerThread; item++) { const int idx = thread_offset + item; out[idx] = thread_data[item]; } } template __global__ void block_scan_single_kernel(T* in, T* out, ActionT action) { using block_scan_t = cub::BlockScan; using storage_t = typename block_scan_t::TempStorage; __shared__ storage_t storage; const int tid = static_cast(cub::RowMajorTid(BlockDimX, BlockDimY, BlockDimZ)); T thread_data = in[tid]; block_scan_t scan(storage); action(scan, thread_data); out[tid] = thread_data; } template void block_scan(c2h::device_vector& in, c2h::device_vector& out, ActionT action) { dim3 block_dims(BlockDimX, BlockDimY, BlockDimZ); block_scan_kernel <<<1, block_dims>>>(thrust::raw_pointer_cast(in.data()), thrust::raw_pointer_cast(out.data()), action); REQUIRE(cudaSuccess == cudaPeekAtLastError()); REQUIRE(cudaSuccess == cudaDeviceSynchronize()); } template void block_scan_single(c2h::device_vector& in, c2h::device_vector& out, ActionT action) { dim3 block_dims(BlockDimX, BlockDimY, BlockDimZ); block_scan_single_kernel <<<1, block_dims>>>(thrust::raw_pointer_cast(in.data()), thrust::raw_pointer_cast(out.data()), action); REQUIRE(cudaSuccess == cudaPeekAtLastError()); REQUIRE(cudaSuccess == cudaDeviceSynchronize()); } enum class scan_mode { exclusive, inclusive }; template struct sum_op_t { template __device__ void operator()(BlockScanT& scan, T (&thread_data)[ItemsPerThread]) const { if constexpr (Mode == scan_mode::exclusive) { scan.ExclusiveSum(thread_data, thread_data); } else { scan.InclusiveSum(thread_data, thread_data); } } }; template struct sum_single_op_t { template __device__ void operator()(BlockScanT& scan, T& thread_data) const { if constexpr (Mode == scan_mode::exclusive) { scan.ExclusiveSum(thread_data, thread_data); } else { scan.InclusiveSum(thread_data, thread_data); } } }; template struct min_init_value_op_t { T initial_value; template __device__ void operator()(BlockScanT& scan, T (&thread_data)[ItemsPerThread]) const { if constexpr (Mode == scan_mode::exclusive) { scan.ExclusiveScan(thread_data, thread_data, initial_value, cuda::minimum<>{}); } else { scan.InclusiveScan(thread_data, thread_data, initial_value, cuda::minimum<>{}); } } }; template struct min_op_t { template __device__ void operator()(BlockScanT& scan, int (&thread_data)[ItemsPerThread]) const { if constexpr (Mode == scan_mode::exclusive) { scan.ExclusiveScan(thread_data, thread_data, cuda::minimum<>{}); } else { scan.InclusiveScan(thread_data, thread_data, cuda::minimum<>{}); } } }; template struct min_init_value_aggregate_op_t { int m_target_thread_id; T initial_value; T* m_d_block_aggregate; template __device__ void operator()(BlockScanT& scan, T (&thread_data)[ItemsPerThread]) const { T block_aggregate{}; if constexpr (Mode == scan_mode::exclusive) { scan.ExclusiveScan(thread_data, thread_data, initial_value, cuda::minimum<>{}, block_aggregate); } else { scan.InclusiveScan(thread_data, thread_data, initial_value, cuda::minimum<>{}, block_aggregate); } const int tid = cub::RowMajorTid(static_cast(blockDim.x), static_cast(blockDim.y), static_cast(blockDim.z)); if (tid == m_target_thread_id) { *m_d_block_aggregate = block_aggregate; } } }; template struct sum_aggregate_op_t { int m_target_thread_id; T* m_d_block_aggregate; template __device__ void operator()(BlockScanT& scan, T (&thread_data)[ItemsPerThread]) const { T block_aggregate{}; if constexpr (Mode == scan_mode::exclusive) { scan.ExclusiveSum(thread_data, thread_data, block_aggregate); } else { scan.InclusiveSum(thread_data, thread_data, block_aggregate); } const int tid = static_cast( cub::RowMajorTid(static_cast(blockDim.x), static_cast(blockDim.y), static_cast(blockDim.z))); if (tid == m_target_thread_id) { *m_d_block_aggregate = block_aggregate; } } }; template struct sum_prefix_op_t { T m_prefix; struct block_prefix_op_t { int linear_tid; T prefix; __device__ block_prefix_op_t(int linear_tid, T prefix) : linear_tid(linear_tid) , prefix(prefix) {} __device__ T operator()(T block_aggregate) { T retval = (linear_tid == 0) ? prefix : T{}; prefix = prefix + block_aggregate; return retval; } }; template __device__ void operator()(BlockScanT& scan, T (&thread_data)[ItemsPerThread]) const { const int tid = static_cast( cub::RowMajorTid(static_cast(blockDim.x), static_cast(blockDim.y), static_cast(blockDim.z))); block_prefix_op_t prefix_op{tid, m_prefix}; if constexpr (Mode == scan_mode::exclusive) { scan.ExclusiveSum(thread_data, thread_data, prefix_op); } else { scan.InclusiveSum(thread_data, thread_data, prefix_op); } } }; template struct min_prefix_op_t { T m_prefix; static constexpr T min_identity = cuda::std::numeric_limits::max(); struct block_prefix_op_t { int linear_tid; T prefix; __device__ block_prefix_op_t(int linear_tid, T prefix) : linear_tid(linear_tid) , prefix(prefix) {} __device__ T operator()(T block_aggregate) { T retval = (linear_tid == 0) ? prefix : min_identity; prefix = cuda::minimum<>{}(prefix, block_aggregate); return retval; } }; template __device__ void operator()(BlockScanT& scan, T (&thread_data)[ItemsPerThread]) const { const int tid = static_cast( cub::RowMajorTid(static_cast(blockDim.x), static_cast(blockDim.y), static_cast(blockDim.z))); block_prefix_op_t prefix_op{tid, m_prefix}; if constexpr (Mode == scan_mode::exclusive) { scan.ExclusiveScan(thread_data, thread_data, cuda::minimum<>{}, prefix_op); } else { scan.InclusiveScan(thread_data, thread_data, cuda::minimum<>{}, prefix_op); } } }; template T host_scan(scan_mode mode, c2h::host_vector& result, ScanOpT scan_op, T initial_value = T{}) { if (result.empty()) { return {}; } T accumulator = static_cast(scan_op(initial_value, result[0])); T block_accumulator = result[0]; if (mode == scan_mode::exclusive) { result[0] = initial_value; for (std::size_t i = 1; i < result.size(); i++) { T tmp = result[i]; result[i] = accumulator; accumulator = static_cast(scan_op(accumulator, tmp)); block_accumulator = static_cast(scan_op(block_accumulator, tmp)); } } else { result[0] = accumulator; for (std::size_t i = 1; i < result.size(); i++) { accumulator = static_cast(scan_op(accumulator, result[i])); block_accumulator = static_cast(scan_op(block_accumulator, result[i])); result[i] = accumulator; } } return block_accumulator; } // %PARAM% ALGO_TYPE alg 0:1:2 // %PARAM% TEST_MODE mode 0:1 using types = c2h::type_list; // FIXME(bgruber): uchar3 fails the test, see #3835 using vec_types = c2h::type_list< #if _CCCL_CTK_AT_LEAST(13, 0) ulonglong4_16a, #else // _CCCL_CTK_AT_LEAST(13, 0) ulonglong4, #endif // _CCCL_CTK_AT_LEAST(13, 0) /*uchar3,*/ short2>; using block_dim_x = c2h::enum_type_list; using block_dim_yz = c2h::enum_type_list; using items_per_thread = c2h::enum_type_list; using single_item_per_thread = c2h::enum_type_list; using algorithms = c2h::enum_type_list; using algorithm = c2h::enum_type_list::value>; #if TEST_MODE == 0 using modes = c2h::enum_type_list; #else using modes = c2h::enum_type_list; #endif template struct params_t { using type = typename c2h::get<0, TestType>; static constexpr int block_dim_x = c2h::get<1, TestType>::value; static constexpr int block_dim_y = c2h::get<2, TestType>::value; static constexpr int block_dim_z = block_dim_y; static constexpr int items_per_thread = c2h::get<3, TestType>::value; static constexpr int tile_size = items_per_thread * block_dim_x * block_dim_y * block_dim_z; static constexpr cub::BlockScanAlgorithm algorithm = c2h::get<4, TestType>::value; static constexpr scan_mode mode = c2h::get<5, TestType>::value; }; C2H_TEST( "Block scan works with sum", "[scan][block]", types, block_dim_x, block_dim_yz, items_per_thread, algorithm, modes) { using params = params_t; using type = typename params::type; c2h::device_vector d_out(params::tile_size); c2h::device_vector d_in(params::tile_size); c2h::gen(C2H_SEED(10), d_in); block_scan( d_in, d_out, sum_op_t{}); c2h::host_vector h_out = d_in; host_scan(params::mode, h_out, std::plus{}); REQUIRE_APPROX_EQ(h_out, d_out); } C2H_TEST("Block scan works with sum single", "[scan][block]", types, block_dim_x, block_dim_yz, single_item_per_thread, algorithm, modes) { using params = params_t; using type = typename params::type; c2h::device_vector d_out(params::tile_size); c2h::device_vector d_in(params::tile_size); c2h::gen(C2H_SEED(10), d_in); block_scan_single( d_in, d_out, sum_single_op_t{}); c2h::host_vector h_out = d_in; host_scan(params::mode, h_out, std::plus{}); REQUIRE_APPROX_EQ(h_out, d_out); } C2H_TEST("Block scan works with vec types", "[scan][block]", vec_types, algorithm, modes) { constexpr int items_per_thread = 3; constexpr int block_dim_x = 256; constexpr int block_dim_y = 1; constexpr int block_dim_z = 1; constexpr int tile_size = items_per_thread * block_dim_x * block_dim_y * block_dim_z; constexpr cub::BlockScanAlgorithm algorithm = c2h::get<1, TestType>::value; constexpr scan_mode mode = c2h::get<2, TestType>::value; using type = typename c2h::get<0, TestType>; c2h::device_vector d_out(tile_size); c2h::device_vector d_in(tile_size); c2h::gen(C2H_SEED(10), d_in); block_scan(d_in, d_out, sum_op_t{}); c2h::host_vector h_out = d_in; host_scan(mode, h_out, std::plus{}); REQUIRE(h_out == d_out); } C2H_TEST("Block scan works with custom types", "[scan][block]", algorithm, modes) { constexpr int items_per_thread = 3; constexpr int block_dim_x = 256; constexpr int block_dim_y = 1; constexpr int block_dim_z = 1; constexpr int tile_size = items_per_thread * block_dim_x * block_dim_y * block_dim_z; constexpr cub::BlockScanAlgorithm algorithm = c2h::get<0, TestType>::value; constexpr scan_mode mode = c2h::get<1, TestType>::value; using type = c2h::custom_type_t; c2h::device_vector d_out(tile_size); c2h::device_vector d_in(tile_size); c2h::gen(C2H_SEED(10), d_in); block_scan(d_in, d_out, sum_op_t{}); c2h::host_vector h_out = d_in; host_scan(mode, h_out, std::plus{}); REQUIRE(h_out == d_out); } C2H_TEST("Block scan returns valid block aggregate", "[scan][block]", algorithm, modes, block_dim_yz) { constexpr int items_per_thread = 3; constexpr int block_dim_x = 64; constexpr int block_dim_y = c2h::get<2, TestType>::value; constexpr int block_dim_z = block_dim_y; constexpr int threads_in_block = block_dim_x * block_dim_y * block_dim_z; constexpr int tile_size = items_per_thread * threads_in_block; constexpr cub::BlockScanAlgorithm algorithm = c2h::get<0, TestType>::value; constexpr scan_mode mode = c2h::get<1, TestType>::value; using type = c2h::custom_type_t; const int target_thread_id = GENERATE_COPY(take(2, random(0, threads_in_block - 1))); c2h::device_vector d_block_aggregate(1); c2h::device_vector d_out(tile_size); c2h::device_vector d_in(tile_size); c2h::gen(C2H_SEED(10), d_in); block_scan( d_in, d_out, sum_aggregate_op_t{target_thread_id, thrust::raw_pointer_cast(d_block_aggregate.data())}); c2h::host_vector h_out = d_in; type block_aggregate = host_scan(mode, h_out, std::plus{}); REQUIRE(h_out == d_out); REQUIRE(block_aggregate == d_block_aggregate[0]); } C2H_TEST("Block scan supports prefix op", "[scan][block]", algorithm, modes, block_dim_yz) { constexpr int items_per_thread = 3; constexpr int block_dim_x = 64; constexpr int block_dim_y = c2h::get<2, TestType>::value; constexpr int block_dim_z = block_dim_y; constexpr int threads_in_block = block_dim_x * block_dim_y * block_dim_z; constexpr int tile_size = items_per_thread * threads_in_block; constexpr cub::BlockScanAlgorithm algorithm = c2h::get<0, TestType>::value; constexpr scan_mode mode = c2h::get<1, TestType>::value; using type = int; const type prefix = GENERATE_COPY(take(2, random(0, tile_size))); c2h::device_vector d_out(tile_size); c2h::device_vector d_in(tile_size); c2h::gen(C2H_SEED(10), d_in); block_scan( d_in, d_out, sum_prefix_op_t{prefix}); c2h::host_vector h_out = d_in; host_scan(mode, h_out, std::plus{}, prefix); REQUIRE(h_out == d_out); } C2H_TEST("Block scan supports custom scan op", "[scan][block]", algorithm, modes, block_dim_yz) { constexpr int items_per_thread = 3; constexpr int block_dim_x = 64; constexpr int block_dim_y = c2h::get<2, TestType>::value; constexpr int block_dim_z = block_dim_y; constexpr int threads_in_block = block_dim_x * block_dim_y * block_dim_z; constexpr int tile_size = items_per_thread * threads_in_block; constexpr cub::BlockScanAlgorithm algorithm = c2h::get<0, TestType>::value; constexpr scan_mode mode = c2h::get<1, TestType>::value; using type = int; c2h::device_vector d_out(tile_size); c2h::device_vector d_in(tile_size); c2h::gen(C2H_SEED(10), d_in); block_scan(d_in, d_out, min_op_t{}); c2h::host_vector h_out = d_in; host_scan( mode, h_out, [](type l, type r) { return std::min(l, r); }, INT_MAX); if constexpr (mode == scan_mode::exclusive) { //! With no initial value, the output computed for *thread*\ :sub:`0` is undefined. d_out.erase(d_out.begin()); h_out.erase(h_out.begin()); } REQUIRE(h_out == d_out); } C2H_TEST("Block custom op scan works with initial value", "[scan][block]", algorithm, modes, block_dim_yz) { constexpr int items_per_thread = 3; constexpr int block_dim_x = 64; constexpr int block_dim_y = c2h::get<2, TestType>::value; constexpr int block_dim_z = block_dim_y; constexpr int threads_in_block = block_dim_x * block_dim_y * block_dim_z; constexpr int tile_size = items_per_thread * threads_in_block; constexpr cub::BlockScanAlgorithm algorithm = c2h::get<0, TestType>::value; constexpr scan_mode mode = c2h::get<1, TestType>::value; using type = int; c2h::device_vector d_out(tile_size); c2h::device_vector d_in(tile_size); c2h::gen(C2H_SEED(10), d_in); const type initial_value = static_cast(GENERATE_COPY(take(2, random(0, tile_size)))); block_scan( d_in, d_out, min_init_value_op_t{initial_value}); c2h::host_vector h_out = d_in; host_scan( mode, h_out, [](type l, type r) { return std::min(l, r); }, initial_value); REQUIRE(h_out == d_out); } C2H_TEST("Block custom op scan with initial value returns valid block aggregate", "[scan][block]", algorithm, modes, block_dim_yz) { constexpr int items_per_thread = 3; constexpr int block_dim_x = 64; constexpr int block_dim_y = c2h::get<2, TestType>::value; constexpr int block_dim_z = block_dim_y; constexpr int threads_in_block = block_dim_x * block_dim_y * block_dim_z; constexpr int tile_size = items_per_thread * threads_in_block; constexpr cub::BlockScanAlgorithm algorithm = c2h::get<0, TestType>::value; constexpr scan_mode mode = c2h::get<1, TestType>::value; using type = int; c2h::device_vector d_out(tile_size); c2h::device_vector d_in(tile_size); c2h::gen(C2H_SEED(10), d_in); const type initial_value = static_cast(GENERATE_COPY(take(2, random(0, tile_size)))); const int target_thread_id = GENERATE_COPY(take(2, random(0, threads_in_block - 1))); c2h::device_vector d_block_aggregate(1); block_scan( d_in, d_out, min_init_value_aggregate_op_t{ target_thread_id, initial_value, thrust::raw_pointer_cast(d_block_aggregate.data())}); c2h::host_vector h_out = d_in; type h_block_aggregate = host_scan( mode, h_out, [](type l, type r) { return std::min(l, r); }, initial_value); REQUIRE(h_out == d_out); REQUIRE(h_block_aggregate == d_block_aggregate[0]); } C2H_TEST("Block scan supports prefix op and custom scan op", "[scan][block]", algorithm, modes, block_dim_yz) { constexpr int items_per_thread = 3; constexpr int block_dim_x = 64; constexpr int block_dim_y = c2h::get<2, TestType>::value; constexpr int block_dim_z = block_dim_y; constexpr int threads_in_block = block_dim_x * block_dim_y * block_dim_z; constexpr int tile_size = items_per_thread * threads_in_block; constexpr cub::BlockScanAlgorithm algorithm = c2h::get<0, TestType>::value; constexpr scan_mode mode = c2h::get<1, TestType>::value; using type = int; const type prefix = GENERATE_COPY(take(2, random(0, tile_size))); c2h::device_vector d_out(tile_size); c2h::device_vector d_in(tile_size); c2h::gen(C2H_SEED(10), d_in); block_scan( d_in, d_out, min_prefix_op_t{prefix}); c2h::host_vector h_out = d_in; host_scan( mode, h_out, [](type a, type b) { return std::min(a, b); }, prefix); REQUIRE(h_out == d_out); }