dylanyunlong/project_6
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

[CCCL] Add missing CCCL components: c2h, nvbench_helper, cmake, cudax, AGENTS.md

Browse Source 
Added 863 files from NVIDIA/cccl sparse checkout:
- c2h/ (27 files): Catch2 test helpers — generators, validators, runner
- nvbench_helper/ (10 files): Benchmark harness utilities
- cmake/ (29 files): CMake presets and build helpers
- cudax/ (794 files): Experimental CUDA extensions
- AGENTS.md: NVIDIA's official AI agent instructions for CCCL
- CMakePresets.json: Standardized build configurations
- cccl-version.json: Version tracking

Also added CCCL_ASSET_MAP.md mapping all 4295 CCCL files to
competition value and PRD items.

cccl_upstream now covers 100% of competition-critical assets:
- 27 tuning headers (SM80/90/100 benchmark data)
- 32 dispatch headers (algorithm implementations)
- 60 Thrust examples (correctness verification)
- 217 CUB Catch2 tests (regression matrix)
- 153 CUB benchmarks (parameter space search)
- 18 CUB examples (API verification)
- 27 test helpers + benchmark harness
- 794 cudax experimental extensions
This commit is contained in:
muh-bot
2026-08-06 02:14:18 +00:00
parent b0d597363a
commit dedf08166a
864 changed files with 174321 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
cccl_upstream/cudax/test/coop/any_of/this_thread.cu Normal file
View File

@@ -0,0 +1,139 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/devices>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/stream>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include "testing.cuh"
template <class Group>
__device__ void test_group(const Group& group)
{
// Single value tests.
{
// Test all threads with false.
{
const auto result = cudax::coop::any_of(group, false);
REQUIRE(result.has_value());
if (result.has_value())
{
REQUIRE(!result.value());
}
}
// Test all threads with true.
{
const auto result = cudax::coop::any_of(group, true);
REQUIRE(result.has_value());
if (result.has_value())
{
REQUIRE(result.value());
}
}
}
{
// Test all threads with false (broadcasted).
{
const bool result = cudax::coop::any_of(cudax::broadcasted, group, false);
REQUIRE(!result);
}
// Test all threads with true (broadcasted).
{
const bool result = cudax::coop::any_of(cudax::broadcasted, group, true);
REQUIRE(result);
}
}
// Array tests.
{
// Test all threads with false.
{
bool in[]{false, false, false};
const auto result = cudax::coop::any_of(group, in);
REQUIRE(result.has_value());
if (result.has_value())
{
REQUIRE(!result.value());
}
}
// Test all threads with true.
{
bool in[]{true, true, true, true};
const auto result = cudax::coop::any_of(group, in);
REQUIRE(result.has_value());
if (result.has_value())
{
REQUIRE(result.value());
}
}
// Test all threads except 1 with false.
{
bool in[]{false, false, true};
const auto result = cudax::coop::any_of(group, in);
REQUIRE(result.has_value());
if (result.has_value())
{
REQUIRE(result.value());
}
}
// Test all threads with false (broadcasted).
{
bool in[]{false, false, false};
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
REQUIRE(!result);
}
// Test all threads with true (broadcasted).
{
bool in[]{true, true, true, true};
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
REQUIRE(result);
}
// Test all threads except 1 with false (broadcasted).
{
bool in[]{false, false, true};
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
REQUIRE(result);
}
}
}
struct TestKernel
{
template <class Config>
__device__ void operator()(const Config& config)
{
test_group(cudax::this_thread{});
test_group(cudax::this_thread{config});
}
};
C2H_TEST("any_of/this_thread", "[any_of][this_thread]")
{
const auto device = cuda::devices[0];
const cuda::stream stream{device};
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
cuda::launch(stream, config, TestKernel{});
stream.sync();
}

156
cccl_upstream/cudax/test/coop/any_of/this_warp.cu Normal file
View File

@@ -0,0 +1,156 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/devices>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/stream>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include "testing.cuh"
template <class Group>
__device__ void test_group(const Group& group)
{
const auto my_rank = cuda::gpu_thread.rank_as<unsigned>(group);
// Single value tests.
{
// Test all threads with false.
{
const auto result = cudax::coop::any_of(group, false);
REQUIRE(result.has_value() == (my_rank == 0));
if (result.has_value())
{
REQUIRE(!result.value());
}
}
// Test all threads with true.
{
const auto result = cudax::coop::any_of(group, true);
REQUIRE(result.has_value() == (my_rank == 0));
if (result.has_value())
{
REQUIRE(result.value());
}
}
// Test all threads except 1 with false.
{
const auto result = cudax::coop::any_of(group, (my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1));
REQUIRE(result.has_value() == (my_rank == 0));
if (result.has_value())
{
REQUIRE(result.value());
}
}
// Test all threads with false (broadcasted).
{
const bool result = cudax::coop::any_of(cudax::broadcasted, group, false);
REQUIRE(!result);
}
// Test all threads with true (broadcasted).
{
const bool result = cudax::coop::any_of(cudax::broadcasted, group, true);
REQUIRE(result);
}
// Test all threads except 1 with false (broadcasted).
{
const bool result =
cudax::coop::any_of(cudax::broadcasted, group, (my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1));
REQUIRE(result);
}
}
// Array tests.
{
// Test all threads with false.
{
bool in[]{false, false, false};
const auto result = cudax::coop::any_of(group, in);
REQUIRE(result.has_value() == (my_rank == 0));
if (result.has_value())
{
REQUIRE(!result.value());
}
}
// Test all threads with true.
{
bool in[]{true, true, true, true};
const auto result = cudax::coop::any_of(group, in);
REQUIRE(result.has_value() == (my_rank == 0));
if (result.has_value())
{
REQUIRE(result.value());
}
}
// Test all threads except 1 with false.
{
bool in[]{false, false, my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1};
const auto result = cudax::coop::any_of(group, in);
REQUIRE(result.has_value() == (my_rank == 0));
if (result.has_value())
{
REQUIRE(result.value());
}
}
// Test all threads with false (broadcasted).
{
bool in[]{false, false, false};
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
REQUIRE(!result);
}
// Test all threads with true (broadcasted).
{
bool in[]{true, true, true, true};
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
REQUIRE(result);
}
// Test all threads except 1 with false (broadcasted).
{
bool in[]{false, false, my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1};
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
REQUIRE(result);
}
}
}
struct TestKernel
{
template <class Config>
__device__ void operator()(const Config& config)
{
test_group(cudax::this_warp{});
test_group(cudax::this_warp{config});
}
};
C2H_TEST("any_of/this_warp", "[any_of][this_warp]")
{
const auto device = cuda::devices[0];
const cuda::stream stream{device};
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
cuda::launch(stream, config, TestKernel{});
stream.sync();
}

187
cccl_upstream/cudax/test/coop/any_of/threads_within_warp.cu Normal file
View File

@@ -0,0 +1,187 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/devices>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/stream>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include "testing.cuh"
template <class Group>
__device__ void test_group(const Group& group)
{
// Exit all threads that are not part of the group.
if (!cuda::gpu_thread.is_part_of(group))
{
return;
}
const auto my_rank = cuda::gpu_thread.rank_as<unsigned>(group);
// Single value tests.
{
// Test all threads with false.
{
const auto result = cudax::coop::any_of(group, false);
REQUIRE(result.has_value() == (my_rank == 0));
if (result.has_value())
{
REQUIRE(!result.value());
}
}
// Test all threads with true.
{
const auto result = cudax::coop::any_of(group, true);
REQUIRE(result.has_value() == (my_rank == 0));
if (result.has_value())
{
REQUIRE(result.value());
}
}
// Test all threads except 1 with false.
{
const auto result = cudax::coop::any_of(group, (my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1));
REQUIRE(result.has_value() == (my_rank == 0));
if (result.has_value())
{
REQUIRE(result.value());
}
}
// Test all threads with false (broadcasted).
{
const bool result = cudax::coop::any_of(cudax::broadcasted, group, false);
REQUIRE(!result);
}
// Test all threads with true (broadcasted).
{
const bool result = cudax::coop::any_of(cudax::broadcasted, group, true);
REQUIRE(result);
}
// Test all threads except 1 with false (broadcasted).
{
const bool result =
cudax::coop::any_of(cudax::broadcasted, group, (my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1));
REQUIRE(result);
}
}
// Array tests.
{
// Test all threads with false.
{
bool in[]{false, false, false};
const auto result = cudax::coop::any_of(group, in);
REQUIRE(result.has_value() == (my_rank == 0));
if (result.has_value())
{
REQUIRE(!result.value());
}
}
// Test all threads with true.
{
bool in[]{true, true, true, true};
const auto result = cudax::coop::any_of(group, in);
REQUIRE(result.has_value() == (my_rank == 0));
if (result.has_value())
{
REQUIRE(result.value());
}
}
// Test all threads except 1 with false.
{
bool in[]{false, false, my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1};
const auto result = cudax::coop::any_of(group, in);
REQUIRE(result.has_value() == (my_rank == 0));
if (result.has_value())
{
REQUIRE(result.value());
}
}
// Test all threads with false (broadcasted).
{
bool in[]{false, false, false};
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
REQUIRE(!result);
}
// Test all threads with true (broadcasted).
{
bool in[]{true, true, true, true};
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
REQUIRE(result);
}
// Test all threads except 1 with false (broadcasted).
{
bool in[]{false, false, my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1};
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
REQUIRE(result);
}
}
}
struct CustomBinaryPartition
{
template <class MappingResult>
__device__ bool operator()(MappingResult mapping_result)
{
switch (mapping_result.unit_rank())
{
case 1:
case 5:
case 14:
case 15:
case 31:
return true;
default:
return false;
}
}
};
struct TestKernel
{
template <class Config>
__device__ void operator()(const Config& config)
{
const cudax::this_warp warp{config};
test_group(cudax::group{cuda::gpu_thread, warp, cudax::identity_mapping{}, cudax::lane_synchronizer{}});
test_group(cudax::group{cuda::gpu_thread, warp, cudax::group_by<4>{}, cudax::lane_synchronizer{}});
test_group(cudax::group{cuda::gpu_thread, warp, cudax::group_by{1}, cudax::lane_synchronizer{}});
test_group(
cudax::group{cuda::gpu_thread, warp, cudax::group_by{3, cudax::non_exhaustive}, cudax::lane_synchronizer{}});
test_group(cudax::group{
cuda::gpu_thread, warp, cudax::binary_partition{CustomBinaryPartition{}}, cudax::lane_synchronizer{}});
}
};
C2H_TEST("any_of/threads_within_warp", "[any_of][threads_within_warp]")
{
const auto device = cuda::devices[0];
const cuda::stream stream{device};
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
cuda::launch(stream, config, TestKernel{});
stream.sync();
}

223
cccl_upstream/cudax/test/coop/reduce/this_block.cu Normal file
View File

@@ -0,0 +1,223 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/devices>
#include <cuda/functional>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/std/algorithm>
#include <cuda/std/type_traits>
#include <cuda/stream>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include <testing.cuh>
#include <c2h/catch2_test_helper.h>
#include <c2h/extended_types.h>
#include <c2h/generators.h>
#include <catch2/matchers/catch_matchers_floating_point.hpp>
/***********************************************************************************************************************
* Thread Reduce Wrapper Kernels
**********************************************************************************************************************/
template <bool Broadcasted>
struct ReduceKernel
{
template <class Config, int NumItems, class T, class RedOp>
__device__ void operator()(
Config config,
cuda::std::integral_constant<int, NumItems>,
const T* __restrict__ d_in,
T* __restrict__ d_out,
RedOp red_op)
{
cudax::this_block block{config};
T thread_data[NumItems];
for (int i = 0; i < NumItems; ++i)
{
thread_data[i] = d_in[cuda::gpu_thread.rank_as<int>(block) + i * cuda::gpu_thread.count_as<int>(block)];
}
if constexpr (Broadcasted)
{
//! [broadcasted reduce]
// Every thread in the block receives the same reduction result.
const auto result = cudax::coop::reduce(cudax::broadcasted, block, thread_data, red_op);
//! [broadcasted reduce]
d_out[cuda::gpu_thread.rank(block)] = result;
}
else
{
const auto result = cudax::coop::reduce(block, thread_data, red_op);
REQUIRE(result.has_value() == cuda::gpu_thread.is_root_rank(block));
if (cuda::gpu_thread.is_root_rank(block))
{
*d_out = result.value();
}
}
}
};
/***********************************************************************************************************************
* Type list definition
**********************************************************************************************************************/
using integral_type_list =
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
using fp_type_list = c2h::type_list<float, double>;
using operator_integral_list =
c2h::type_list<cuda::std::plus<>,
cuda::std::multiplies<>,
cuda::std::bit_and<>,
cuda::std::bit_or<>,
cuda::std::bit_xor<>,
cuda::minimum<>,
cuda::maximum<>>;
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
using block_size_list = c2h::enum_type_list<int, 3, 32, 63, 128>;
/***********************************************************************************************************************
* Verify results and kernel launch
**********************************************************************************************************************/
template <class T>
void verify_results(const T& expected_data, const T& test_results)
{
if constexpr (cuda::std::is_floating_point_v<T>)
{
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
}
else
{
REQUIRE(expected_data == test_results);
}
}
template <int BlockSize, class T, class RedOp, bool Broadcasted = false>
void run_reduce_kernel(
cuda::stream_ref stream,
cuda::std::integral_constant<int, BlockSize>,
int num_items,
const c2h::device_vector<T>& in,
c2h::device_vector<T>& out,
RedOp red_op,
cuda::std::bool_constant<Broadcasted> = {})
{
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<BlockSize>());
const auto in_ptr = thrust::raw_pointer_cast(in.data());
const auto out_ptr = thrust::raw_pointer_cast(out.data());
const ReduceKernel<Broadcasted> kernel{};
switch (num_items)
{
case 1:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 1>{}, in_ptr, out_ptr, red_op);
break;
case 4:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 4>{}, in_ptr, out_ptr, red_op);
break;
default:
FAIL("Unsupported number of items");
}
stream.sync();
}
constexpr int max_size = 4;
constexpr int num_seeds = 10;
/***********************************************************************************************************************
* Test cases
**********************************************************************************************************************/
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
C2H_TEST("reduce/this_block Integral Type Tests",
"[reduce][this_block]",
integral_type_list,
operator_integral_list,
block_size_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
using block_size_t = c2h::get<2, TestType>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * block_size_t::value);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items : {1, 4})
{
auto reference_result =
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * block_size_t::value, operator_identity, reduce_op);
run_reduce_kernel(stream, block_size_t{}, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST("reduce/this_block Floating-Point Type Tests",
"[reduce][this_block]",
fp_type_list,
operator_fp_list,
block_size_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
using block_size_t = c2h::get<2, TestType>;
constexpr auto reduce_op = op_t{};
const auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * block_size_t::value);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items : {1, 4})
{
auto reference_result =
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * block_size_t::value, operator_identity, reduce_op);
run_reduce_kernel(stream, block_size_t{}, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST("reduce/this_block Broadcasted", "[reduce][this_block]", integral_type_list, block_size_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = cuda::std::plus<>;
using block_size_t = c2h::get<1, TestType>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * block_size_t::value);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items : {1, 4})
{
c2h::device_vector<value_t> d_out(block_size_t::value);
auto reference_result =
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * block_size_t::value, operator_identity, reduce_op);
run_reduce_kernel(stream, block_size_t{}, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
verify_results(c2h::host_vector<value_t>(block_size_t::value, reference_result), c2h::host_vector<value_t>(d_out));
}
}

242
cccl_upstream/cudax/test/coop/reduce/this_cluster.cu Normal file
View File

@@ -0,0 +1,242 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/devices>
#include <cuda/functional>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/std/algorithm>
#include <cuda/std/type_traits>
#include <cuda/stream>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include <testing.cuh>
#include <c2h/catch2_test_helper.h>
#include <c2h/extended_types.h>
#include <c2h/generators.h>
#include <catch2/matchers/catch_matchers_floating_point.hpp>
constexpr int block_size = 64;
/***********************************************************************************************************************
* Thread Reduce Wrapper Kernels
**********************************************************************************************************************/
template <bool Broadcasted>
struct ReduceKernel
{
template <class Config, int NumItems, class T, class RedOp>
__device__ void operator()(
Config config,
cuda::std::integral_constant<int, NumItems>,
const T* __restrict__ d_in,
T* __restrict__ d_out,
RedOp red_op)
{
cudax::this_cluster cluster{config};
T thread_data[NumItems];
for (int i = 0; i < NumItems; ++i)
{
thread_data[i] = d_in[cuda::gpu_thread.rank_as<int>(cluster) + i * cuda::gpu_thread.count_as<int>(cluster)];
}
if constexpr (Broadcasted)
{
const auto result = cudax::coop::reduce(cudax::broadcasted, cluster, thread_data, red_op);
d_out[cuda::gpu_thread.rank(cluster)] = result;
}
else
{
const auto result = cudax::coop::reduce(cluster, thread_data, red_op);
REQUIRE(result.has_value() == cuda::gpu_thread.is_root_rank(cluster));
if (cuda::gpu_thread.is_root_rank(cluster))
{
*d_out = result.value();
}
}
}
};
/***********************************************************************************************************************
* Type list definition
**********************************************************************************************************************/
using integral_type_list =
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
using fp_type_list = c2h::type_list<float, double>;
using operator_integral_list =
c2h::type_list<cuda::std::plus<>,
cuda::std::multiplies<>,
cuda::std::bit_and<>,
cuda::std::bit_or<>,
cuda::std::bit_xor<>,
cuda::minimum<>,
cuda::maximum<>>;
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
using cluster_size_list = c2h::enum_type_list<int, 1, 2, 4, 8>;
/***********************************************************************************************************************
* Verify results and kernel launch
**********************************************************************************************************************/
template <class T>
void verify_results(const T& expected_data, const T& test_results)
{
if constexpr (cuda::std::is_floating_point_v<T>)
{
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
}
else
{
REQUIRE(expected_data == test_results);
}
}
template <int ClusterSize, class T, class RedOp, bool Broadcasted = false>
void run_reduce_kernel(
cuda::stream_ref stream,
cuda::std::integral_constant<int, ClusterSize>,
int num_items,
const c2h::device_vector<T>& in,
c2h::device_vector<T>& out,
RedOp red_op,
cuda::std::bool_constant<Broadcasted> = {})
{
const auto config =
cuda::make_config(cuda::grid_dims<1>(), cuda::cluster_dims<ClusterSize>(), cuda::block_dims<block_size>());
const auto in_ptr = thrust::raw_pointer_cast(in.data());
const auto out_ptr = thrust::raw_pointer_cast(out.data());
const ReduceKernel<Broadcasted> kernel{};
switch (num_items)
{
case 1:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 1>{}, in_ptr, out_ptr, red_op);
break;
case 4:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 4>{}, in_ptr, out_ptr, red_op);
break;
default:
FAIL("Unsupported number of items");
}
stream.sync();
}
constexpr int max_size = 4;
constexpr int num_seeds = 10;
/***********************************************************************************************************************
* Test cases
**********************************************************************************************************************/
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
C2H_TEST("reduce/this_cluster Integral Type Tests",
"[reduce][this_cluster]",
integral_type_list,
operator_integral_list,
cluster_size_list)
{
const auto device = cuda::devices[0];
if (cuda::device_attributes::compute_capability_major(device) < 9)
{
return;
}
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
using cluster_size_t = c2h::get<2, TestType>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * cluster_size_t::value * block_size);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{device};
for (int num_items : {1, 4})
{
auto reference_result = cuda::std::accumulate(
h_in.begin(), h_in.begin() + num_items * cluster_size_t::value * block_size, operator_identity, reduce_op);
run_reduce_kernel(stream, cluster_size_t{}, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST("reduce/this_cluster Floating-Point Type Tests",
"[reduce][this_cluster]",
fp_type_list,
operator_fp_list,
cluster_size_list)
{
const auto device = cuda::devices[0];
if (cuda::device_attributes::compute_capability_major(device) < 9)
{
return;
}
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
using cluster_size_t = c2h::get<2, TestType>;
constexpr auto reduce_op = op_t{};
const auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * cluster_size_t::value * block_size);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{device};
for (int num_items : {1, 4})
{
auto reference_result = cuda::std::accumulate(
h_in.begin(), h_in.begin() + num_items * cluster_size_t::value * block_size, operator_identity, reduce_op);
run_reduce_kernel(stream, cluster_size_t{}, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST("reduce/this_cluster Broadcasted", "[reduce][this_cluster]", integral_type_list, cluster_size_list)
{
const auto device = cuda::devices[0];
if (cuda::device_attributes::compute_capability_major(device) < 9)
{
return;
}
using value_t = c2h::get<0, TestType>;
using op_t = cuda::std::plus<>;
using cluster_size_t = c2h::get<1, TestType>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * cluster_size_t::value * block_size);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{device};
for (int num_items : {1, 4})
{
c2h::device_vector<value_t> d_out(cluster_size_t::value * block_size);
auto reference_result = cuda::std::accumulate(
h_in.begin(), h_in.begin() + num_items * cluster_size_t::value * block_size, operator_identity, reduce_op);
run_reduce_kernel(stream, cluster_size_t{}, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
verify_results(c2h::host_vector<value_t>(cluster_size_t::value * block_size, reference_result),
c2h::host_vector<value_t>(d_out));
}
}

252
cccl_upstream/cudax/test/coop/reduce/this_grid.cu Normal file
View File

@@ -0,0 +1,252 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/devices>
#include <cuda/functional>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/std/algorithm>
#include <cuda/std/type_traits>
#include <cuda/stream>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include <testing.cuh>
#include <c2h/catch2_test_helper.h>
#include <c2h/extended_types.h>
#include <c2h/generators.h>
#include <catch2/matchers/catch_matchers_floating_point.hpp>
constexpr int cluster_size = 2;
constexpr int block_size = 128;
/***********************************************************************************************************************
* Thread Reduce Wrapper Kernels
**********************************************************************************************************************/
template <bool Broadcasted>
struct ReduceKernel
{
template <class Config, int NumItems, class T, class RedOp>
__device__ void operator()(
Config config,
cuda::std::integral_constant<int, NumItems>,
const T* __restrict__ d_in,
T* __restrict__ d_out,
RedOp red_op)
{
cudax::this_grid grid{config};
T thread_data[NumItems];
for (int i = 0; i < NumItems; ++i)
{
thread_data[i] = d_in[cuda::gpu_thread.rank_as<int>(grid) + i * cuda::gpu_thread.count_as<int>(grid)];
}
if constexpr (Broadcasted)
{
const auto result = cudax::coop::reduce(cudax::broadcasted, grid, thread_data, red_op);
d_out[cuda::gpu_thread.rank(grid)] = result;
}
else
{
const auto result = cudax::coop::reduce(grid, thread_data, red_op);
REQUIRE(result.has_value() == cuda::gpu_thread.is_root_rank(grid));
if (cuda::gpu_thread.is_root_rank(grid))
{
*d_out = result.value();
}
}
}
};
/***********************************************************************************************************************
* Type list definition
**********************************************************************************************************************/
using integral_type_list =
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
using fp_type_list = c2h::type_list<float, double>;
using operator_integral_list =
c2h::type_list<cuda::std::plus<>,
cuda::std::multiplies<>,
cuda::std::bit_and<>,
cuda::std::bit_or<>,
cuda::std::bit_xor<>,
cuda::minimum<>,
cuda::maximum<>>;
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
using grid_size_list = c2h::enum_type_list<int, 1, 12, 32>;
/***********************************************************************************************************************
* Verify results and kernel launch`
**********************************************************************************************************************/
template <class T>
void verify_results(const T& expected_data, const T& test_results)
{
if constexpr (cuda::std::is_floating_point_v<T>)
{
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
}
else
{
REQUIRE(expected_data == test_results);
}
}
template <int GridSize, class T, class RedOp, bool Broadcasted = false>
void run_reduce_kernel(
cuda::stream_ref stream,
cuda::std::integral_constant<int, GridSize>,
int num_items,
const c2h::device_vector<T>& in,
c2h::device_vector<T>& out,
RedOp red_op,
cuda::std::bool_constant<Broadcasted> = {})
{
const auto config = cuda::make_config(
cuda::grid_dims<GridSize>(),
cuda::cluster_dims<cluster_size>(),
cuda::block_dims<block_size>(),
cuda::cooperative_launch{});
const auto in_ptr = thrust::raw_pointer_cast(in.data());
const auto out_ptr = thrust::raw_pointer_cast(out.data());
const ReduceKernel<Broadcasted> kernel{};
switch (num_items)
{
case 1:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 1>{}, in_ptr, out_ptr, red_op);
break;
case 4:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 4>{}, in_ptr, out_ptr, red_op);
break;
default:
FAIL("Unsupported number of items");
}
stream.sync();
}
constexpr int max_size = 4;
constexpr int num_seeds = 10;
/***********************************************************************************************************************
* Test cases
**********************************************************************************************************************/
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
C2H_TEST("reduce/this_grid Integral Type Tests",
"[reduce][this_grid]",
integral_type_list,
operator_integral_list,
grid_size_list)
{
const auto device = cuda::devices[0];
if (cuda::device_attributes::compute_capability_major(device) < 9)
{
return;
}
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
using grid_size_t = c2h::get<2, TestType>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * grid_size_t::value * cluster_size * block_size);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{device};
for (int num_items : {1, 4})
{
auto reference_result = cuda::std::accumulate(
h_in.begin(),
h_in.begin() + num_items * grid_size_t::value * cluster_size * block_size,
operator_identity,
reduce_op);
run_reduce_kernel(stream, grid_size_t{}, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST(
"reduce/this_grid Floating-Point Type Tests", "[reduce][this_grid]", fp_type_list, operator_fp_list, grid_size_list)
{
const auto device = cuda::devices[0];
if (cuda::device_attributes::compute_capability_major(device) < 9)
{
return;
}
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
using grid_size_t = c2h::get<2, TestType>;
constexpr auto reduce_op = op_t{};
const auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * grid_size_t::value * cluster_size * block_size);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{device};
for (int num_items : {1, 4})
{
auto reference_result = cuda::std::accumulate(
h_in.begin(),
h_in.begin() + num_items * grid_size_t::value * cluster_size * block_size,
operator_identity,
reduce_op);
run_reduce_kernel(stream, grid_size_t{}, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST("reduce/this_grid Broadcasted", "[reduce][this_grid]", integral_type_list, grid_size_list)
{
const auto device = cuda::devices[0];
if (cuda::device_attributes::compute_capability_major(device) < 9)
{
return;
}
using value_t = c2h::get<0, TestType>;
using op_t = cuda::std::plus<>;
using grid_size_t = c2h::get<1, TestType>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * grid_size_t::value * cluster_size * block_size);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{device};
for (int num_items : {1, 4})
{
c2h::device_vector<value_t> d_out(grid_size_t::value * cluster_size * block_size);
auto reference_result = cuda::std::accumulate(
h_in.begin(),
h_in.begin() + num_items * grid_size_t::value * cluster_size * block_size,
operator_identity,
reduce_op);
run_reduce_kernel(stream, grid_size_t{}, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
verify_results(c2h::host_vector<value_t>(grid_size_t::value * cluster_size * block_size, reference_result),
c2h::host_vector<value_t>(d_out));
}
}

242
cccl_upstream/cudax/test/coop/reduce/this_thread.cu Normal file
View File

@@ -0,0 +1,242 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/devices>
#include <cuda/functional>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/std/algorithm>
#include <cuda/std/type_traits>
#include <cuda/stream>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include <testing.cuh>
#include <c2h/catch2_test_helper.h>
#include <c2h/extended_types.h>
#include <c2h/generators.h>
#include <catch2/matchers/catch_matchers_floating_point.hpp>
/***********************************************************************************************************************
* Thread Reduce Wrapper Kernels
**********************************************************************************************************************/
template <bool Broadcasted>
struct ReduceKernel
{
template <class Config, int NumItems, class T, class RedOp>
__device__ void operator()(
Config config,
cuda::std::integral_constant<int, NumItems>,
const T* __restrict__ d_in,
T* __restrict__ d_out,
RedOp red_op)
{
cudax::this_thread thread{config};
T thread_data[NumItems];
for (int i = 0; i < NumItems; ++i)
{
thread_data[i] = d_in[i];
}
if constexpr (Broadcasted)
{
const auto result = cudax::coop::reduce(cudax::broadcasted, thread, thread_data, red_op);
*d_out = result;
}
else
{
const auto result = cudax::coop::reduce(thread, thread_data, red_op);
REQUIRE(result.has_value());
*d_out = result.value();
}
}
};
/***********************************************************************************************************************
* Type list definition
**********************************************************************************************************************/
using integral_type_list =
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
using fp_type_list = c2h::type_list<float, double>;
using operator_integral_list =
c2h::type_list<cuda::std::plus<>,
cuda::std::multiplies<>,
cuda::std::bit_and<>,
cuda::std::bit_or<>,
cuda::std::bit_xor<>,
cuda::minimum<>,
cuda::maximum<>>;
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
/***********************************************************************************************************************
* Verify results and kernel launch
**********************************************************************************************************************/
template <class T>
void verify_results(const T& expected_data, const T& test_results)
{
if constexpr (cuda::std::is_floating_point_v<T>)
{
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
}
else
{
REQUIRE(expected_data == test_results);
}
}
template <class T, class RedOp, bool Broadcasted = false>
void run_reduce_kernel(
cuda::stream_ref stream,
int num_items,
const c2h::device_vector<T>& in,
c2h::device_vector<T>& out,
RedOp red_op,
cuda::std::bool_constant<Broadcasted> = {})
{
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<1>());
const auto in_ptr = thrust::raw_pointer_cast(in.data());
const auto out_ptr = thrust::raw_pointer_cast(out.data());
const ReduceKernel<Broadcasted> kernel{};
switch (num_items)
{
case 1:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 1>{}, in_ptr, out_ptr, red_op);
break;
case 2:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 2>{}, in_ptr, out_ptr, red_op);
break;
case 3:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 3>{}, in_ptr, out_ptr, red_op);
break;
case 4:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 4>{}, in_ptr, out_ptr, red_op);
break;
case 5:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 5>{}, in_ptr, out_ptr, red_op);
break;
case 6:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 6>{}, in_ptr, out_ptr, red_op);
break;
case 7:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 7>{}, in_ptr, out_ptr, red_op);
break;
case 8:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 8>{}, in_ptr, out_ptr, red_op);
break;
case 9:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 9>{}, in_ptr, out_ptr, red_op);
break;
case 10:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 10>{}, in_ptr, out_ptr, red_op);
break;
case 11:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 11>{}, in_ptr, out_ptr, red_op);
break;
case 12:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 12>{}, in_ptr, out_ptr, red_op);
break;
case 13:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 13>{}, in_ptr, out_ptr, red_op);
break;
case 14:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 14>{}, in_ptr, out_ptr, red_op);
break;
case 15:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 15>{}, in_ptr, out_ptr, red_op);
break;
case 16:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 16>{}, in_ptr, out_ptr, red_op);
break;
default:
FAIL("Unsupported number of items");
}
stream.sync();
}
constexpr int max_size = 16;
constexpr int num_seeds = 10;
/***********************************************************************************************************************
* Test cases
**********************************************************************************************************************/
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
C2H_TEST("reduce/this_thread Integral Type Tests", "[reduce][this_thread]", integral_type_list, operator_integral_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items = 1; num_items <= max_size; ++num_items)
{
auto reference_result = cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items, operator_identity, reduce_op);
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST("reduce/this_thread Floating-Point Type Tests", "[reduce][this_thread]", fp_type_list, operator_fp_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
constexpr auto reduce_op = op_t{};
const auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items = 1; num_items <= max_size; ++num_items)
{
auto reference_result = cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items, operator_identity, reduce_op);
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST("reduce/this_thread Broadcasted", "[reduce][this_thread]", integral_type_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = cuda::std::plus<>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items = 1; num_items <= max_size; ++num_items)
{
auto reference_result = cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items, operator_identity, reduce_op);
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}

213
cccl_upstream/cudax/test/coop/reduce/this_warp.cu Normal file
View File

@@ -0,0 +1,213 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/devices>
#include <cuda/functional>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/std/algorithm>
#include <cuda/std/type_traits>
#include <cuda/stream>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include <testing.cuh>
#include <c2h/catch2_test_helper.h>
#include <c2h/extended_types.h>
#include <c2h/generators.h>
#include <catch2/matchers/catch_matchers_floating_point.hpp>
/***********************************************************************************************************************
* Thread Reduce Wrapper Kernels
**********************************************************************************************************************/
template <bool Broadcasted>
struct ReduceKernel
{
template <class Config, int NumItems, class T, class RedOp>
__device__ void operator()(
Config config,
cuda::std::integral_constant<int, NumItems>,
const T* __restrict__ d_in,
T* __restrict__ d_out,
RedOp red_op)
{
cudax::this_warp warp{config};
T thread_data[NumItems];
for (int i = 0; i < NumItems; ++i)
{
thread_data[i] = d_in[cuda::gpu_thread.rank_as<int>(warp) + i * cuda::gpu_thread.count_as<int>(warp)];
}
if constexpr (Broadcasted)
{
const auto result = cudax::coop::reduce(cudax::broadcasted, warp, thread_data, red_op);
d_out[cuda::gpu_thread.rank(warp)] = result;
}
else
{
const auto result = cudax::coop::reduce(warp, thread_data, red_op);
REQUIRE(result.has_value() == cuda::gpu_thread.is_root_rank(warp));
if (cuda::gpu_thread.is_root_rank(warp))
{
*d_out = result.value();
}
}
}
};
/***********************************************************************************************************************
* Type list definition
**********************************************************************************************************************/
using integral_type_list =
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
using fp_type_list = c2h::type_list<float, double>;
using operator_integral_list =
c2h::type_list<cuda::std::plus<>,
cuda::std::multiplies<>,
cuda::std::bit_and<>,
cuda::std::bit_or<>,
cuda::std::bit_xor<>,
cuda::minimum<>,
cuda::maximum<>>;
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
/***********************************************************************************************************************
* Verify results and kernel launch
**********************************************************************************************************************/
template <class T>
void verify_results(const T& expected_data, const T& test_results)
{
if constexpr (cuda::std::is_floating_point_v<T>)
{
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
}
else
{
REQUIRE(expected_data == test_results);
}
}
template <class T, class RedOp, bool Broadcasted = false>
void run_reduce_kernel(
cuda::stream_ref stream,
int num_items,
const c2h::device_vector<T>& in,
c2h::device_vector<T>& out,
RedOp red_op,
cuda::std::bool_constant<Broadcasted> = {})
{
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
const auto in_ptr = thrust::raw_pointer_cast(in.data());
const auto out_ptr = thrust::raw_pointer_cast(out.data());
const ReduceKernel<Broadcasted> kernel{};
switch (num_items)
{
case 1:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 1>{}, in_ptr, out_ptr, red_op);
break;
case 2:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 2>{}, in_ptr, out_ptr, red_op);
break;
case 3:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 3>{}, in_ptr, out_ptr, red_op);
break;
case 4:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 4>{}, in_ptr, out_ptr, red_op);
break;
default:
FAIL("Unsupported number of items");
}
stream.sync();
}
constexpr int warp_size = 32;
constexpr int max_size = 4;
constexpr int num_seeds = 10;
/***********************************************************************************************************************
* Test cases
**********************************************************************************************************************/
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
C2H_TEST("reduce/this_warp Integral Type Tests", "[reduce][this_warp]", integral_type_list, operator_integral_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * warp_size);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items = 1; num_items <= max_size; ++num_items)
{
auto reference_result =
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * warp_size, operator_identity, reduce_op);
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST("reduce/this_warp Floating-Point Type Tests", "[reduce][this_warp]", fp_type_list, operator_fp_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
constexpr auto reduce_op = op_t{};
const auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * warp_size);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items = 1; num_items <= max_size; ++num_items)
{
auto reference_result =
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * warp_size, operator_identity, reduce_op);
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST("reduce/this_warp Broadcasted", "[reduce][this_warp]", integral_type_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = ::cuda::std::plus<>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * warp_size);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items = 1; num_items <= max_size; ++num_items)
{
c2h::device_vector<value_t> d_out(warp_size);
auto reference_result =
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * warp_size, operator_identity, reduce_op);
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
verify_results(c2h::host_vector<value_t>(warp_size, reference_result), c2h::host_vector<value_t>(d_out));
}
}

279
cccl_upstream/cudax/test/coop/reduce/threads_within_warp.cu Normal file
View File

@@ -0,0 +1,279 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/atomic>
#include <cuda/devices>
#include <cuda/functional>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/std/algorithm>
#include <cuda/std/type_traits>
#include <cuda/stream>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include <testing.cuh>
#include <c2h/catch2_test_helper.h>
#include <c2h/extended_types.h>
#include <c2h/generators.h>
#include <catch2/matchers/catch_matchers_floating_point.hpp>
constexpr int warp_size = 32;
/***********************************************************************************************************************
* Thread Reduce Wrapper Kernels
**********************************************************************************************************************/
template <bool Broadcasted>
struct ReduceKernel
{
template <class Config, unsigned NThreadsInGroup, int NumItems, class T, class RedOp>
__device__ void operator()(
Config config,
cuda::std::integral_constant<unsigned, NThreadsInGroup>,
cuda::std::integral_constant<int, NumItems>,
const T* __restrict__ d_in,
T* __restrict__ d_out,
RedOp red_op)
{
cudax::group group{
cuda::gpu_thread, cudax::this_warp{config}, cudax::group_by<NThreadsInGroup, false>{}, cudax::lane_synchronizer{}};
// All threads that are not part of the groups should exit early.
if (!cuda::gpu_thread.is_part_of(group))
{
return;
}
// We want to work only with one group, all other groups should exit early.
if (group.rank(cuda::warp) > 0)
{
return;
}
T thread_data[NumItems];
for (int i = 0; i < NumItems; ++i)
{
thread_data[i] = d_in[cuda::gpu_thread.rank_as<int>(group) + i * cuda::gpu_thread.count_as<int>(group)];
}
if constexpr (Broadcasted)
{
const auto result = cudax::coop::reduce(cudax::broadcasted, group, thread_data, red_op);
d_out[cuda::gpu_thread.rank(group)] = result;
}
else
{
const auto result = cudax::coop::reduce(group, thread_data, red_op);
REQUIRE(result.has_value() == cuda::gpu_thread.is_root_rank(group));
if (cuda::gpu_thread.is_root_rank(group))
{
*d_out = result.value();
}
}
}
};
/***********************************************************************************************************************
* Type list definition
**********************************************************************************************************************/
using integral_type_list =
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
using fp_type_list = c2h::type_list<float, double>;
using operator_integral_list =
c2h::type_list<cuda::std::plus<>,
cuda::std::multiplies<>,
cuda::std::bit_and<>,
cuda::std::bit_or<>,
cuda::std::bit_xor<>,
cuda::minimum<>,
cuda::maximum<>>;
using nthreads_in_group_list = c2h::enum_type_list<unsigned, 1, 2, 12, 31, 32>;
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
/***********************************************************************************************************************
* Verify results and kernel launch
**********************************************************************************************************************/
template <class T>
void verify_results(const T& expected_data, const T& test_results)
{
if constexpr (cuda::std::is_floating_point_v<T>)
{
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
}
else
{
REQUIRE(expected_data == test_results);
}
}
template <unsigned NThreadsInGroup, class T, class RedOp, bool Broadcasted = false>
void run_reduce_kernel(
cuda::stream_ref stream,
cuda::std::integral_constant<unsigned, NThreadsInGroup>,
int num_items,
const c2h::device_vector<T>& in,
c2h::device_vector<T>& out,
RedOp red_op,
cuda::std::bool_constant<Broadcasted> = {})
{
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<warp_size>());
const auto in_ptr = thrust::raw_pointer_cast(in.data());
const auto out_ptr = thrust::raw_pointer_cast(out.data());
const ReduceKernel<Broadcasted> kernel{};
switch (num_items)
{
case 1:
cuda::launch(
stream,
config,
kernel,
cuda::std::integral_constant<unsigned, NThreadsInGroup>{},
cuda::std::integral_constant<int, 1>{},
in_ptr,
out_ptr,
red_op);
break;
case 2:
cuda::launch(
stream,
config,
kernel,
cuda::std::integral_constant<unsigned, NThreadsInGroup>{},
cuda::std::integral_constant<int, 2>{},
in_ptr,
out_ptr,
red_op);
break;
case 3:
cuda::launch(
stream,
config,
kernel,
cuda::std::integral_constant<unsigned, NThreadsInGroup>{},
cuda::std::integral_constant<int, 3>{},
in_ptr,
out_ptr,
red_op);
break;
case 4:
cuda::launch(
stream,
config,
kernel,
cuda::std::integral_constant<unsigned, NThreadsInGroup>{},
cuda::std::integral_constant<int, 4>{},
in_ptr,
out_ptr,
red_op);
break;
default:
FAIL("Unsupported number of items");
}
stream.sync();
}
constexpr int max_size = 4;
constexpr int num_seeds = 10;
/***********************************************************************************************************************
* Test cases
**********************************************************************************************************************/
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
C2H_TEST("reduce/threads_within_warp Integral Type Tests",
"[reduce][threads_within_warp]",
integral_type_list,
operator_integral_list,
nthreads_in_group_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
using nthreads_in_group_t = c2h::get<2, TestType>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
constexpr auto nthreads_in_group = nthreads_in_group_t::value;
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * nthreads_in_group);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items = 1; num_items <= max_size; ++num_items)
{
auto reference_result =
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * nthreads_in_group, operator_identity, reduce_op);
run_reduce_kernel(stream, nthreads_in_group_t{}, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST("reduce/threads_within_warp Floating-Point Type Tests",
"[reduce][threads_within_warp]",
fp_type_list,
operator_fp_list,
nthreads_in_group_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
using nthreads_in_group_t = c2h::get<2, TestType>;
constexpr auto reduce_op = op_t{};
constexpr auto nthreads_in_group = nthreads_in_group_t::value;
const auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * nthreads_in_group);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items = 1; num_items <= max_size; ++num_items)
{
auto reference_result =
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * nthreads_in_group, operator_identity, reduce_op);
run_reduce_kernel(stream, nthreads_in_group_t{}, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST(
"reduce/threads_within_warp Broadcasted", "[reduce][threads_within_warp]", integral_type_list, nthreads_in_group_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = cuda::std::plus<>;
using nthreads_in_group_t = c2h::get<1, TestType>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
constexpr auto nthreads_in_group = nthreads_in_group_t::value;
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * nthreads_in_group);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items = 1; num_items <= max_size; ++num_items)
{
c2h::device_vector<value_t> d_out(nthreads_in_group);
auto reference_result =
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * nthreads_in_group, operator_identity, reduce_op);
run_reduce_kernel(stream, nthreads_in_group_t{}, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
verify_results(c2h::host_vector<value_t>(nthreads_in_group, reference_result), c2h::host_vector<value_t>(d_out));
}
}

234
cccl_upstream/cudax/test/coop/reduce/warps_within_block.cu Normal file
View File

@@ -0,0 +1,234 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/atomic>
#include <cuda/devices>
#include <cuda/functional>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/std/algorithm>
#include <cuda/std/type_traits>
#include <cuda/stream>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include <testing.cuh>
#include <c2h/catch2_test_helper.h>
#include <c2h/extended_types.h>
#include <c2h/generators.h>
#include <catch2/matchers/catch_matchers_floating_point.hpp>
constexpr int nwarps_in_group = 3;
constexpr int warp_size = 32;
/***********************************************************************************************************************
* Thread Reduce Wrapper Kernels
**********************************************************************************************************************/
template <bool Broadcasted>
struct ReduceKernel
{
template <class Config, int NumItems, class T, class RedOp>
__device__ void operator()(
Config config,
cuda::std::integral_constant<int, NumItems>,
const T* __restrict__ d_in,
T* __restrict__ d_out,
RedOp red_op)
{
cudax::this_block block{config};
using Barriers = cuda::barrier<cuda::thread_scope_block>[1];
__shared__ cuda::std::aligned_storage_t<sizeof(Barriers), alignof(Barriers)> barriers_storage;
auto& barriers = reinterpret_cast<Barriers&>(barriers_storage);
cudax::group group{
cuda::warp, block, cudax::group_by<nwarps_in_group, false>{}, cudax::barrier_synchronizer{barriers}};
// All threads that are not part of the groups should exit early.
if (!cuda::gpu_thread.is_part_of(group))
{
return;
}
T thread_data[NumItems];
for (int i = 0; i < NumItems; ++i)
{
thread_data[i] = d_in[cuda::gpu_thread.rank_as<int>(group) + i * cuda::gpu_thread.count_as<int>(group)];
}
if constexpr (Broadcasted)
{
const auto result = cudax::coop::reduce(cudax::broadcasted, group, thread_data, red_op);
d_out[cuda::gpu_thread.rank(group)] = result;
}
else
{
const auto result = cudax::coop::reduce(group, thread_data, red_op);
REQUIRE(result.has_value() == cuda::gpu_thread.is_root_rank(group));
if (cuda::gpu_thread.is_root_rank(group))
{
*d_out = result.value();
}
}
}
};
/***********************************************************************************************************************
* Type list definition
**********************************************************************************************************************/
using integral_type_list =
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
using fp_type_list = c2h::type_list<float, double>;
using operator_integral_list =
c2h::type_list<cuda::std::plus<>,
cuda::std::multiplies<>,
cuda::std::bit_and<>,
cuda::std::bit_or<>,
cuda::std::bit_xor<>,
cuda::minimum<>,
cuda::maximum<>>;
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
/***********************************************************************************************************************
* Verify results and kernel launch
**********************************************************************************************************************/
template <class T>
void verify_results(const T& expected_data, const T& test_results)
{
if constexpr (cuda::std::is_floating_point_v<T>)
{
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
}
else
{
REQUIRE(expected_data == test_results);
}
}
template <class T, class RedOp, bool Broadcasted = false>
void run_reduce_kernel(
cuda::stream_ref stream,
int num_items,
const c2h::device_vector<T>& in,
c2h::device_vector<T>& out,
RedOp red_op,
cuda::std::bool_constant<Broadcasted> = {})
{
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<(nwarps_in_group + 2) * warp_size>());
const auto in_ptr = thrust::raw_pointer_cast(in.data());
const auto out_ptr = thrust::raw_pointer_cast(out.data());
const ReduceKernel<Broadcasted> kernel{};
switch (num_items)
{
case 1:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 1>{}, in_ptr, out_ptr, red_op);
break;
case 2:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 2>{}, in_ptr, out_ptr, red_op);
break;
case 3:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 3>{}, in_ptr, out_ptr, red_op);
break;
case 4:
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 4>{}, in_ptr, out_ptr, red_op);
break;
default:
FAIL("Unsupported number of items");
}
stream.sync();
}
constexpr int max_size = 4;
constexpr int num_seeds = 10;
/***********************************************************************************************************************
* Test cases
**********************************************************************************************************************/
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
C2H_TEST("reduce/warps_within_block Integral Type Tests",
"[reduce][warps_within_block]",
integral_type_list,
operator_integral_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * nwarps_in_group * warp_size);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items = 1; num_items <= max_size; ++num_items)
{
auto reference_result = cuda::std::accumulate(
h_in.begin(), h_in.begin() + num_items * nwarps_in_group * warp_size, operator_identity, reduce_op);
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST(
"reduce/warps_within_block Floating-Point Type Tests", "[reduce][warps_within_block]", fp_type_list, operator_fp_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = c2h::get<1, TestType>;
constexpr auto reduce_op = op_t{};
const auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * nwarps_in_group * warp_size);
c2h::device_vector<value_t> d_out(1);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items = 1; num_items <= max_size; ++num_items)
{
auto reference_result = cuda::std::accumulate(
h_in.begin(), h_in.begin() + num_items * nwarps_in_group * warp_size, operator_identity, reduce_op);
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op);
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
}
}
C2H_TEST("reduce/warps_within_block Broadcasted", "[reduce][warps_within_block]", integral_type_list)
{
using value_t = c2h::get<0, TestType>;
using op_t = cuda::std::plus<>;
constexpr auto reduce_op = op_t{};
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
c2h::device_vector<value_t> d_in(max_size * nwarps_in_group * warp_size);
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
c2h::host_vector<value_t> h_in = d_in;
cuda::stream stream{cuda::devices[0]};
for (int num_items = 1; num_items <= max_size; ++num_items)
{
c2h::device_vector<value_t> d_out(nwarps_in_group * warp_size);
auto reference_result = cuda::std::accumulate(
h_in.begin(), h_in.begin() + num_items * nwarps_in_group * warp_size, operator_identity, reduce_op);
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
verify_results(c2h::host_vector<value_t>(nwarps_in_group * warp_size, reference_result),
c2h::host_vector<value_t>(d_out));
}
}

274
cccl_upstream/cudax/test/coop/scratch.cu Normal file
View File

@@ -0,0 +1,274 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/__functional/call_or.h>
#include <cuda/__functional/lazy_call_or.h>
#include <cuda/devices>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/std/execution>
#include <cuda/std/type_traits>
#include <cuda/stream>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include "testing.cuh"
enum class MyCoopAlgScratch
{
none,
smem,
gmem,
smem_gmem,
};
template <MyCoopAlgScratch Kind>
using MyCoopAlgScratchConstant = cuda::std::integral_constant<MyCoopAlgScratch, Kind>;
struct MyCoopAlgSmemScratch
{
char data[128];
};
struct MyCoopAlgGmemScratch
{
char data[128];
};
template <bool _False = false>
__device__ void my_coop_alg_impl(...)
{
static_assert(_False, "This group type is unsupported by MyCoopAlg");
}
template <class Group, MyCoopAlgScratch Kind, class SmemScratch, class GmemScratch>
__device__ void my_coop_alg_impl(const Group&, MyCoopAlgScratchConstant<Kind>, SmemScratch&, GmemScratch&)
{
// algorithm implementation
}
struct MyCoopAlg
{
template <class Group, MyCoopAlgScratch Kind, class Env = cuda::std::execution::env<>>
[[nodiscard]] __device__ static constexpr auto
__get_scratch_requirements(const Group& group, MyCoopAlgScratchConstant<Kind>, Env = {}) noexcept
{
if constexpr (Kind == MyCoopAlgScratch::none)
{
return cudax::coop::__scratch_reqs<cudax::coop::__empty_smem_scratch, cudax::coop::__empty_gmem_scratch>{};
}
else if constexpr (Kind == MyCoopAlgScratch::smem)
{
return cudax::coop::__scratch_reqs<MyCoopAlgSmemScratch, cudax::coop::__empty_gmem_scratch>{};
}
else if constexpr (Kind == MyCoopAlgScratch::gmem)
{
return cudax::coop::__scratch_reqs<cudax::coop::__empty_smem_scratch, MyCoopAlgGmemScratch>{};
}
else
{
return cudax::coop::__scratch_reqs<MyCoopAlgSmemScratch, MyCoopAlgGmemScratch>{};
}
}
template <class Group, MyCoopAlgScratch Kind, class Env = cuda::std::execution::env<>>
__device__ void operator()(const Group& group, MyCoopAlgScratchConstant<Kind> kind, Env env = {}) const
{
// Get reference scratch requirements for this parameter combination.
using ScratchReqs = decltype(cudax::coop::get_scratch_requirements(MyCoopAlg{}, group, kind, env));
using ExpSmemScratch = typename ScratchReqs::shared_memory_type;
using ExpGmemScratch = typename ScratchReqs::global_memory_type;
// Check that environment's smem and gmem scratch match the expected type.
if constexpr (cuda::std::execution::__queryable_with<Env, cudax::coop::__get_smem_scratch_t>)
{
using QueryResult =
cuda::std::remove_cvref_t<cuda::std::execution::__query_result_t<Env, cudax::coop::__get_smem_scratch_t>>;
using EnvSmemScratch = typename QueryResult::type;
static_assert(cuda::std::is_same_v<EnvSmemScratch, ExpSmemScratch>, "Invalid shared memory scratch passed");
}
if constexpr (cuda::std::execution::__queryable_with<Env, cudax::coop::__get_gmem_scratch_t>)
{
using QueryResult =
cuda::std::remove_cvref_t<cuda::std::execution::__query_result_t<Env, cudax::coop::__get_gmem_scratch_t>>;
using EnvGmemScratch = typename QueryResult::type;
static_assert(cuda::std::is_same_v<EnvGmemScratch, ExpGmemScratch>, "Invalid global memory scratch passed");
}
else
{
static_assert(!ScratchReqs::needs_global_memory, "Algorithm can't allocate global memory for scratch by itself");
}
// Extract environment's scratch or allocate default scratch.
auto& smem_scratch =
cuda::__lazy_call_or(
cudax::coop::__get_smem_scratch,
[&]() {
return cudax::coop::__make_smem_scratch<ExpSmemScratch>(group, kind, env);
},
env)
.get();
auto& gmem_scratch =
cuda::__call_or(
cudax::coop::__get_gmem_scratch, cuda::std::reference_wrapper{cudax::coop::__empty_gmem_scratch_obj}, env)
.get();
// Pass scratch to algorithm implementation.
my_coop_alg_impl(group, kind, smem_scratch, gmem_scratch);
}
};
__device__ constexpr MyCoopAlg my_coop_alg;
__device__ cudax::coop::__empty_gmem_scratch my_empty_gmem_scratch;
__device__ MyCoopAlgGmemScratch my_gmem_scratch;
struct TestKernel
{
template <class Config>
__device__ void operator()(Config config) const
{
__shared__ cudax::coop::__empty_smem_scratch my_empty_smem_scratch;
__shared__ MyCoopAlgSmemScratch my_smem_scratch;
const cudax::this_thread group{config};
// Test no scratch requirements.
{
const MyCoopAlgScratchConstant<MyCoopAlgScratch::none> kind{};
using ScratchReqs = decltype(cudax::coop::get_scratch_requirements(my_coop_alg, group, kind));
static_assert(cuda::std::is_same_v<typename ScratchReqs::shared_memory_type, cudax::coop::__empty_smem_scratch>);
static_assert(cuda::std::is_same_v<typename ScratchReqs::global_memory_type, cudax::coop::__empty_gmem_scratch>);
static_assert(!ScratchReqs::needs_shared_memory);
static_assert(!ScratchReqs::needs_global_memory);
static_assert(ScratchReqs::shared_memory_size == 0);
static_assert(ScratchReqs::global_memory_size == 0);
static_assert(ScratchReqs::shared_memory_alignment == 0);
static_assert(ScratchReqs::global_memory_alignment == 0);
// Test default environment.
my_coop_alg(group, kind);
// Test custom smem scratch.
my_coop_alg(group, kind, cuda::std::execution::env{cudax::coop::shared_memory_scratch(my_empty_smem_scratch)});
// Test custom gmem scratch.
my_coop_alg(group, kind, cuda::std::execution::env{cudax::coop::global_memory_scratch(my_empty_gmem_scratch)});
// Test custom smem and gmem scratch.
my_coop_alg(group,
kind,
cuda::std::execution::env{cudax::coop::shared_memory_scratch(my_empty_smem_scratch),
cudax::coop::global_memory_scratch(my_empty_gmem_scratch)});
}
// Test smem scratch requirements.
{
const MyCoopAlgScratchConstant<MyCoopAlgScratch::smem> kind{};
using ScratchReqs = decltype(cudax::coop::get_scratch_requirements(my_coop_alg, group, kind));
static_assert(cuda::std::is_same_v<typename ScratchReqs::shared_memory_type, MyCoopAlgSmemScratch>);
static_assert(cuda::std::is_same_v<typename ScratchReqs::global_memory_type, cudax::coop::__empty_gmem_scratch>);
static_assert(ScratchReqs::needs_shared_memory);
static_assert(!ScratchReqs::needs_global_memory);
static_assert(ScratchReqs::shared_memory_size == sizeof(MyCoopAlgSmemScratch));
static_assert(ScratchReqs::global_memory_size == 0);
static_assert(ScratchReqs::shared_memory_alignment == alignof(MyCoopAlgSmemScratch));
static_assert(ScratchReqs::global_memory_alignment == 0);
// Test default environment.
my_coop_alg(group, kind);
// Test custom smem scratch.
my_coop_alg(group, kind, cuda::std::execution::env{cudax::coop::shared_memory_scratch(my_smem_scratch)});
// Test custom gmem scratch.
my_coop_alg(group, kind, cuda::std::execution::env{cudax::coop::global_memory_scratch(my_empty_gmem_scratch)});
// Test custom smem and gmem scratch.
my_coop_alg(group,
kind,
cuda::std::execution::env{cudax::coop::shared_memory_scratch(my_smem_scratch),
cudax::coop::global_memory_scratch(my_empty_gmem_scratch)});
}
// Test gmem scratch requirements.
{
const MyCoopAlgScratchConstant<MyCoopAlgScratch::gmem> kind{};
using ScratchReqs = decltype(cudax::coop::get_scratch_requirements(my_coop_alg, group, kind));
static_assert(cuda::std::is_same_v<typename ScratchReqs::shared_memory_type, cudax::coop::__empty_smem_scratch>);
static_assert(cuda::std::is_same_v<typename ScratchReqs::global_memory_type, MyCoopAlgGmemScratch>);
static_assert(!ScratchReqs::needs_shared_memory);
static_assert(ScratchReqs::needs_global_memory);
static_assert(ScratchReqs::shared_memory_size == 0);
static_assert(ScratchReqs::global_memory_size == sizeof(MyCoopAlgGmemScratch));
static_assert(ScratchReqs::shared_memory_alignment == 0);
static_assert(ScratchReqs::global_memory_alignment == alignof(MyCoopAlgGmemScratch));
// Test custom gmem scratch.
my_coop_alg(group, kind, cuda::std::execution::env{cudax::coop::global_memory_scratch(my_gmem_scratch)});
// Test custom smem and gmem scratch.
my_coop_alg(group,
kind,
cuda::std::execution::env{cudax::coop::shared_memory_scratch(my_empty_smem_scratch),
cudax::coop::global_memory_scratch(my_gmem_scratch)});
}
// Test smem and gmem scratch requirements.
{
const MyCoopAlgScratchConstant<MyCoopAlgScratch::smem_gmem> kind{};
using ScratchReqs = decltype(cudax::coop::get_scratch_requirements(my_coop_alg, group, kind));
static_assert(cuda::std::is_same_v<typename ScratchReqs::shared_memory_type, MyCoopAlgSmemScratch>);
static_assert(cuda::std::is_same_v<typename ScratchReqs::global_memory_type, MyCoopAlgGmemScratch>);
static_assert(ScratchReqs::needs_shared_memory);
static_assert(ScratchReqs::needs_global_memory);
static_assert(ScratchReqs::shared_memory_size == sizeof(MyCoopAlgSmemScratch));
static_assert(ScratchReqs::global_memory_size == sizeof(MyCoopAlgGmemScratch));
static_assert(ScratchReqs::shared_memory_alignment == alignof(MyCoopAlgSmemScratch));
static_assert(ScratchReqs::global_memory_alignment == alignof(MyCoopAlgGmemScratch));
// Test custom gmem scratch.
my_coop_alg(group, kind, cuda::std::execution::env{cudax::coop::global_memory_scratch(my_gmem_scratch)});
// Test custom smem and gmem scratch.
my_coop_alg(group,
kind,
cuda::std::execution::env{cudax::coop::shared_memory_scratch(my_smem_scratch),
cudax::coop::global_memory_scratch(my_gmem_scratch)});
}
}
};
C2H_TEST("scratch", "[scratch]")
{
const auto device = cuda::devices[0];
cuda::stream stream{device};
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<1>());
cuda::launch(stream, config, TestKernel{});
stream.sync();
}

141
cccl_upstream/cudax/test/coop/shuffle/threads_within_warp.cu Normal file
View File

@@ -0,0 +1,141 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/devices>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/stream>
#include <cuda/type_traits>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include "testing.cuh"
template <class T>
__device__ T make_instance_for(unsigned rank)
{
if constexpr (cuda::is_vector_type_v<T>)
{
using U = cuda::scalar_type_t<T>;
return T{static_cast<U>(rank), static_cast<U>(rank), static_cast<U>(rank)};
}
else
{
return static_cast<T>(rank);
}
}
template <class T, class Group>
__device__ void test_group(const Group& group)
{
// Exit all threads that are not part of the group.
if (!cuda::gpu_thread.is_part_of(group))
{
return;
}
const auto my_rank = cuda::gpu_thread.rank_as<unsigned>(group);
const auto my_value = make_instance_for<T>(my_rank);
// Test identity.
REQUIRE(cudax::coop::shuffle(group, my_value, my_rank) == my_value);
// Test broadcast from root rank.
REQUIRE(cudax::coop::shuffle(group, my_value, 0) == make_instance_for<T>(0));
// Test broadcast from last rank.
{
const auto other_rank = cuda::gpu_thread.count_as<unsigned>(group) - 1;
REQUIRE(cudax::coop::shuffle(group, my_value, other_rank) == make_instance_for<T>(other_rank));
}
// Test my_rank + 1.
{
const auto other_rank = (my_rank + 1) % cuda::gpu_thread.count_as<unsigned>(group);
REQUIRE(cudax::coop::shuffle(group, my_value, other_rank) == make_instance_for<T>(other_rank));
}
// Test my_rank + 6.
{
const auto other_rank = (my_rank + 6) % cuda::gpu_thread.count_as<unsigned>(group);
REQUIRE(cudax::coop::shuffle(group, my_value, other_rank) == make_instance_for<T>(other_rank));
}
// Test my_rank +- 1 based on whether my_rank is even or not.
{
const auto other_rank =
((my_rank % 2 == 0) ? my_rank + 1 : my_rank - 1) % cuda::gpu_thread.count_as<unsigned>(group);
REQUIRE(cudax::coop::shuffle(group, my_value, other_rank) == make_instance_for<T>(other_rank));
}
// Test my_rank +- 3 based on whether my_rank is even or not.
{
const auto other_rank =
((my_rank % 2 == 0) ? my_rank + 3 : my_rank - 3) % cuda::gpu_thread.count_as<unsigned>(group);
REQUIRE(cudax::coop::shuffle(group, my_value, other_rank) == make_instance_for<T>(other_rank));
}
}
struct CustomBinaryPartition
{
template <class MappingResult>
__device__ bool operator()(MappingResult mapping_result)
{
switch (mapping_result.unit_rank())
{
case 1:
case 5:
case 14:
case 15:
case 31:
return true;
default:
return false;
}
}
};
template <class T, class Config>
__device__ void test_type(const Config& config)
{
const cudax::this_warp warp{config};
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::identity_mapping{}, cudax::lane_synchronizer{}});
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::group_by<4>{}, cudax::lane_synchronizer{}});
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::group_by{1}, cudax::lane_synchronizer{}});
test_group<T>(
cudax::group{cuda::gpu_thread, warp, cudax::group_by{3, cudax::non_exhaustive}, cudax::lane_synchronizer{}});
test_group<T>(
cudax::group{cuda::gpu_thread, warp, cudax::binary_partition{CustomBinaryPartition{}}, cudax::lane_synchronizer{}});
}
struct TestKernel
{
template <class Config>
__device__ void operator()(const Config& config)
{
test_type<signed char>(config);
test_type<unsigned>(config);
test_type<unsigned long long>(config);
test_type<longlong3>(config);
}
};
C2H_TEST("shuffle/threads_within_warp", "[shuffle][threads_within_warp]")
{
const auto device = cuda::devices[0];
const cuda::stream stream{device};
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
cuda::launch(stream, config, TestKernel{});
stream.sync();
}

143
cccl_upstream/cudax/test/coop/shuffle_down/threads_within_warp.cu Normal file
View File

@@ -0,0 +1,143 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/devices>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/stream>
#include <cuda/type_traits>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include "testing.cuh"
template <class T>
__device__ T make_instance_for(unsigned rank)
{
if constexpr (cuda::is_vector_type_v<T>)
{
using U = cuda::scalar_type_t<T>;
return T{static_cast<U>(rank), static_cast<U>(rank), static_cast<U>(rank)};
}
else
{
return static_cast<T>(rank);
}
}
template <class T, class Group>
__device__ void test_group(const Group& group)
{
// Exit all threads that are not part of the group.
if (!cuda::gpu_thread.is_part_of(group))
{
return;
}
const auto my_rank = cuda::gpu_thread.rank_as<unsigned>(group);
const auto my_value = make_instance_for<T>(my_rank);
// Test identity.
REQUIRE(cudax::coop::shuffle_down(group, my_value, 0) == my_value);
// Test getting value from the next rank.
{
const auto offset = 1;
const auto other_rank = cuda::gpu_thread.rank(group) + offset;
const auto ref = (other_rank < cuda::gpu_thread.count(group))
? cuda::std::optional{make_instance_for<T>(other_rank)}
: cuda::std::nullopt;
REQUIRE(cudax::coop::shuffle_down(group, my_value, offset) == ref);
}
// Test getting value from the this + 2 rank.
{
const auto offset = 2;
const auto other_rank = cuda::gpu_thread.rank(group) + offset;
const auto ref = (other_rank < cuda::gpu_thread.count(group))
? cuda::std::optional{make_instance_for<T>(other_rank)}
: cuda::std::nullopt;
REQUIRE(cudax::coop::shuffle_down(group, my_value, offset) == ref);
}
// Test getting value from the last rank.
{
const auto other_rank = cuda::gpu_thread.count(group) - 1;
const auto offset = other_rank - cuda::gpu_thread.rank(group);
REQUIRE(cudax::coop::shuffle_down(group, my_value, offset) == make_instance_for<T>(other_rank));
}
// Test getting value from the last rank + 1.
{
const auto other_rank = cuda::gpu_thread.count(group);
const auto offset = other_rank - cuda::gpu_thread.rank(group);
REQUIRE(cudax::coop::shuffle_down(group, my_value, offset) == cuda::std::nullopt);
}
// Test getting value from out of range offset.
REQUIRE(cudax::coop::shuffle_down(group, my_value, ~0u) == cuda::std::nullopt);
}
struct CustomBinaryPartition
{
template <class MappingResult>
__device__ bool operator()(MappingResult mapping_result)
{
switch (mapping_result.unit_rank())
{
case 1:
case 5:
case 14:
case 15:
case 31:
return true;
default:
return false;
}
}
};
template <class T, class Config>
__device__ void test_type(const Config& config)
{
const cudax::this_warp warp{config};
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::identity_mapping{}, cudax::lane_synchronizer{}});
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::group_by<4>{}, cudax::lane_synchronizer{}});
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::group_by{1}, cudax::lane_synchronizer{}});
test_group<T>(
cudax::group{cuda::gpu_thread, warp, cudax::group_by{3, cudax::non_exhaustive}, cudax::lane_synchronizer{}});
test_group<T>(
cudax::group{cuda::gpu_thread, warp, cudax::binary_partition{CustomBinaryPartition{}}, cudax::lane_synchronizer{}});
}
struct TestKernel
{
template <class Config>
__device__ void operator()(const Config& config)
{
test_type<signed char>(config);
test_type<unsigned>(config);
test_type<unsigned long long>(config);
test_type<longlong3>(config);
}
};
C2H_TEST("shuffle/threads_within_warp", "[shuffle][threads_within_warp]")
{
const auto device = cuda::devices[0];
const cuda::stream stream{device};
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
cuda::launch(stream, config, TestKernel{});
stream.sync();
}

142
cccl_upstream/cudax/test/coop/shuffle_up/threads_within_warp.cu Normal file
View File

@@ -0,0 +1,142 @@
//===----------------------------------------------------------------------===//
//
// Part of CUDA Experimental in CUDA C++ Core Libraries,
// under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
//
//===----------------------------------------------------------------------===//
#include <cuda/devices>
#include <cuda/hierarchy>
#include <cuda/launch>
#include <cuda/stream>
#include <cuda/type_traits>
#include <cuda/experimental/coop.cuh>
#include <cuda/experimental/group.cuh>
#include "testing.cuh"
template <class T>
__device__ T make_instance_for(unsigned rank)
{
if constexpr (cuda::is_vector_type_v<T>)
{
using U = cuda::scalar_type_t<T>;
return T{static_cast<U>(rank), static_cast<U>(rank), static_cast<U>(rank)};
}
else
{
return static_cast<T>(rank);
}
}
template <class T, class Group>
__device__ void test_group(const Group& group)
{
// Exit all threads that are not part of the group.
if (!cuda::gpu_thread.is_part_of(group))
{
return;
}
const auto my_rank = cuda::gpu_thread.rank_as<unsigned>(group);
const auto my_value = make_instance_for<T>(my_rank);
// Test identity.
REQUIRE(cudax::coop::shuffle_up(group, my_value, 0) == my_value);
// Test getting value from the previous rank.
{
const auto offset = 1;
const auto other_rank = cuda::gpu_thread.rank(group) - offset;
const auto ref = (other_rank < cuda::gpu_thread.count(group))
? cuda::std::optional{make_instance_for<T>(other_rank)}
: cuda::std::nullopt;
REQUIRE(cudax::coop::shuffle_up(group, my_value, offset) == ref);
}
// Test getting value from the this + 2 rank.
{
const auto offset = 2;
const auto other_rank = cuda::gpu_thread.rank(group) - offset;
const auto ref = (other_rank < cuda::gpu_thread.count(group))
? cuda::std::optional{make_instance_for<T>(other_rank)}
: cuda::std::nullopt;
REQUIRE(cudax::coop::shuffle_up(group, my_value, offset) == ref);
}
// Test getting value from the first rank.
{
const auto other_rank = 0;
const auto offset = cuda::gpu_thread.rank(group) - other_rank;
REQUIRE(cudax::coop::shuffle_up(group, my_value, offset) == make_instance_for<T>(other_rank));
}
// Test getting value from the first rank - 1.
{
const auto offset = cuda::gpu_thread.rank(group) + 1;
REQUIRE(cudax::coop::shuffle_up(group, my_value, offset) == cuda::std::nullopt);
}
// Test getting value from out of range offset.
REQUIRE(cudax::coop::shuffle_up(group, my_value, ~0u) == cuda::std::nullopt);
}
struct CustomBinaryPartition
{
template <class MappingResult>
__device__ bool operator()(MappingResult mapping_result)
{
switch (mapping_result.unit_rank())
{
case 1:
case 5:
case 14:
case 15:
case 31:
return true;
default:
return false;
}
}
};
template <class T, class Config>
__device__ void test_type(const Config& config)
{
const cudax::this_warp warp{config};
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::identity_mapping{}, cudax::lane_synchronizer{}});
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::group_by<4>{}, cudax::lane_synchronizer{}});
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::group_by{1}, cudax::lane_synchronizer{}});
test_group<T>(
cudax::group{cuda::gpu_thread, warp, cudax::group_by{3, cudax::non_exhaustive}, cudax::lane_synchronizer{}});
test_group<T>(
cudax::group{cuda::gpu_thread, warp, cudax::binary_partition{CustomBinaryPartition{}}, cudax::lane_synchronizer{}});
}
struct TestKernel
{
template <class Config>
__device__ void operator()(const Config& config)
{
test_type<signed char>(config);
test_type<unsigned>(config);
test_type<unsigned long long>(config);
test_type<longlong3>(config);
}
};
C2H_TEST("shuffle/threads_within_warp", "[shuffle][threads_within_warp]")
{
const auto device = cuda::devices[0];
const cuda::stream stream{device};
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
cuda::launch(stream, config, TestKernel{});
stream.sync();
}