[CCCL] 瘦身 + 补全: 移除 cudax/python/libcudacxx-tests 冗余文件, 新增 c2h 测试助手 + cmake 构建系统 + 8 个 CUDA thrust examples
变更摘要:
- 删除: cudax/ (783 files, 7.2M) — 实验性组件，竞赛不需要
- 删除: python/ (226 files, 2.0M) — Python 绑定，竞赛不需要
- 删除: libcudacxx/{test,benchmarks,codegen,cmake,share} (4432 files, 31M)
保留: libcudacxx/include/ (1463 headers, cuda::std 编译依赖)
- 新增: c2h/ (27 files) — CUB Catch2 测试辅助头文件，编译 243 个测试必需
- 新增: cmake/ (29 files) — CCCL 原生 CMake 构建系统
- 新增: thrust/examples/cuda/ (7 files) + cpp_integration/ (1 file)
async_reduce, custom_temporary_allocation, explicit_cuda_stream,
global_device_vector, range_view, unwrap_pointer, wrap_pointer, device
结果: cccl_upstream 从 74M→35M (瘦身 53%), 核心内容 100% 保留:
27/27 tuning headers, 78 benchmarks, 243 tests,
60 thrust examples, 18 CUB examples, 全部编译头文件
This commit is contained in:
muh-bot
@@ -1,218 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cub/block/block_reduce.cuh>
|
||||
#include <cub/thread/thread_reduce.cuh>
|
||||
#include <cub/warp/warp_reduce.cuh>
|
||||
|
||||
#include <cuda/atomic>
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/optional>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include <cooperative_groups.h>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
template <class Hierarchy, class T, cuda::std::size_t N>
|
||||
__device__ cuda::std::optional<T> sum(cudax::this_thread<Hierarchy> group, T (&array)[N])
|
||||
{
|
||||
return {cub::ThreadReduce(array, cuda::std::plus<T>{})};
|
||||
}
|
||||
|
||||
template <class Hierarchy, class T, cuda::std::size_t N>
|
||||
__device__ cuda::std::optional<T> sum(cudax::this_warp<Hierarchy> group, T (&array)[N])
|
||||
{
|
||||
using WarpReduce = cub::WarpReduce<T>;
|
||||
|
||||
__shared__ typename WarpReduce::TempStorage scratch;
|
||||
|
||||
const auto partial = cub::ThreadReduce(array, cuda::std::plus<T>{});
|
||||
const auto result = WarpReduce{scratch}.Sum(partial);
|
||||
return (cuda::gpu_thread.is_root_rank(group)) ? cuda::std::optional{result} : cuda::std::nullopt;
|
||||
}
|
||||
|
||||
template <class Hierarchy, class T, cuda::std::size_t N>
|
||||
__device__ cuda::std::optional<T> sum(cudax::this_block<Hierarchy> group, T (&array)[N])
|
||||
{
|
||||
using BlockExts = decltype(cuda::gpu_thread.extents(cuda::block, group.hierarchy()));
|
||||
static_assert(BlockExts::rank_dynamic() == 0, "This algorithm requires all static extents.");
|
||||
|
||||
using BlockReduce =
|
||||
cub::BlockReduce<T,
|
||||
static_cast<int>(BlockExts::static_extent(0)),
|
||||
cub::BLOCK_REDUCE_WARP_REDUCTIONS,
|
||||
static_cast<int>(BlockExts::static_extent(1)),
|
||||
static_cast<int>(BlockExts::static_extent(2))>;
|
||||
|
||||
__shared__ typename BlockReduce::TempStorage scratch;
|
||||
const auto result = BlockReduce{scratch}.Sum(array);
|
||||
return (cuda::gpu_thread.is_root_rank(group)) ? cuda::std::optional{result} : cuda::std::nullopt;
|
||||
}
|
||||
|
||||
template <class Hierarchy, class T, cuda::std::size_t N>
|
||||
__device__ cuda::std::optional<T> sum(cudax::this_cluster<Hierarchy> group, T (&array)[N])
|
||||
{
|
||||
using BlockExts = decltype(cuda::gpu_thread.extents(cuda::block, group.hierarchy()));
|
||||
static_assert(BlockExts::rank_dynamic() == 0, "This algorithm requires all static extents.");
|
||||
|
||||
using BlockReduce =
|
||||
cub::BlockReduce<T,
|
||||
static_cast<int>(BlockExts::static_extent(0)),
|
||||
cub::BLOCK_REDUCE_WARP_REDUCTIONS,
|
||||
static_cast<int>(BlockExts::static_extent(1)),
|
||||
static_cast<int>(BlockExts::static_extent(2))>;
|
||||
|
||||
union SMem
|
||||
{
|
||||
typename BlockReduce::TempStorage block_scratch;
|
||||
T cluster_scratch;
|
||||
};
|
||||
|
||||
__shared__ SMem smem;
|
||||
T result = BlockReduce{smem.block_scratch}.Sum(array);
|
||||
|
||||
NV_IF_TARGET(NV_PROVIDES_SM_90, ({
|
||||
const auto dsmem = static_cast<T*>(__cluster_map_shared_rank(&smem.cluster_scratch, 0));
|
||||
|
||||
if (cuda::gpu_thread.is_root_rank(group))
|
||||
{
|
||||
smem.cluster_scratch = result;
|
||||
}
|
||||
group.sync_aligned();
|
||||
|
||||
cudax::this_block this_block{group.hierarchy()};
|
||||
if (cuda::gpu_thread.is_root_rank(this_block) && !cuda::gpu_thread.is_root_rank(group))
|
||||
{
|
||||
[[maybe_unused]] unsigned old;
|
||||
asm volatile("atom.relaxed.cluster.shared::cluster.add.s32 %0, [%1], %2;"
|
||||
: "=r"(old)
|
||||
: "l"(dsmem), "r"(result)
|
||||
: "memory");
|
||||
}
|
||||
group.sync_aligned();
|
||||
|
||||
if (cuda::gpu_thread.is_root_rank(group))
|
||||
{
|
||||
result = smem.cluster_scratch;
|
||||
}
|
||||
}))
|
||||
|
||||
return (cuda::gpu_thread.is_root_rank(group)) ? cuda::std::optional{result} : cuda::std::nullopt;
|
||||
}
|
||||
|
||||
// todo(dabayer): Add support for warp and cluster levels.
|
||||
template <class Group, class T, cuda::std::size_t N>
|
||||
__device__ cuda::std::optional<T> sum(Group group, T (&array)[N])
|
||||
{
|
||||
using Unit = typename Group::unit_type;
|
||||
using MappingResult = typename Group::__mapping_result_type;
|
||||
|
||||
constexpr auto ngroups = MappingResult::static_group_count();
|
||||
static_assert(ngroups != cuda::std::dynamic_extent, "group count must be statically known");
|
||||
|
||||
__shared__ T group_sums[ngroups];
|
||||
|
||||
if (!Unit{}.is_part_of(group))
|
||||
{
|
||||
return cuda::std::nullopt;
|
||||
}
|
||||
|
||||
// todo(dabayer): Replace by group.rank(level) once this query is available.
|
||||
const auto group_rank = group.__mapping_result().group_rank();
|
||||
|
||||
if (cuda::gpu_thread.is_root_rank(group))
|
||||
{
|
||||
group_sums[group_rank] = 0;
|
||||
}
|
||||
|
||||
const auto unit_group = cudax::make_this_group(Unit{}, group.hierarchy());
|
||||
const auto result_unit = sum(unit_group, array);
|
||||
|
||||
// Wait until group_sums are are filled with 0.
|
||||
group.sync_aligned();
|
||||
|
||||
if (cuda::gpu_thread.is_root_rank(unit_group))
|
||||
{
|
||||
cuda::atomic_ref<T, cuda::thread_scope_block>{group_sums[group_rank]} += result_unit.value();
|
||||
}
|
||||
|
||||
// Wait until all unit_group roots add the intermediate sum to the shared memory.
|
||||
group.sync_aligned();
|
||||
|
||||
return (cuda::gpu_thread.is_root_rank(group)) ? cuda::std::optional{group_sums[group_rank]} : cuda::std::nullopt;
|
||||
}
|
||||
|
||||
template <class Group>
|
||||
__device__ void test_cooperative_algorithm(Group group)
|
||||
{
|
||||
using Level = typename Group::level_type;
|
||||
|
||||
unsigned array[]{1, 2, 3};
|
||||
const auto result = sum(group, array);
|
||||
|
||||
const auto ref_sum = static_cast<unsigned>(6 * cuda::gpu_thread.count(group));
|
||||
|
||||
// Only the root rank should have the correct result.
|
||||
if (cuda::gpu_thread.is_root_rank(group))
|
||||
{
|
||||
REQUIRE(result.has_value());
|
||||
REQUIRE(result == ref_sum);
|
||||
}
|
||||
else
|
||||
{
|
||||
REQUIRE(!result.has_value());
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_cooperative_algorithm(cudax::this_thread{config});
|
||||
test_cooperative_algorithm(cudax::this_warp{config});
|
||||
test_cooperative_algorithm(cudax::this_block{config});
|
||||
test_cooperative_algorithm(cudax::this_cluster{config});
|
||||
|
||||
test_cooperative_algorithm(
|
||||
cudax::group{cuda::gpu_thread, cudax::this_block{config}, cudax::group_by<2>{}, cudax::lane_synchronizer{}});
|
||||
test_cooperative_algorithm(
|
||||
cudax::group{cuda::gpu_thread, cudax::this_block{config}, cudax::group_by<16>{}, cudax::lane_synchronizer{}});
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("Collective algorithm", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
const auto config = cuda::make_config(cuda::grid_dims<2>(), cuda::block_dims<128>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
|
||||
if (cuda::device_attributes::compute_capability(device) >= cuda::compute_capability{90})
|
||||
{
|
||||
const auto config_cluster =
|
||||
cuda::make_config(cuda::grid_dims<2>(), cuda::cluster_dims<3>(), cuda::block_dims<128>());
|
||||
cuda::launch(stream, config_cluster, TestKernel{});
|
||||
}
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,191 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/atomic>
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
__device__ unsigned global_var = 0;
|
||||
|
||||
template <class Unit, class Level, class Hierarchy, class Group>
|
||||
__device__ void test_common_properties(const Hierarchy&, Group& group)
|
||||
{
|
||||
// Assert that Group satisfies the group concept.
|
||||
static_assert(cudax::is_group<Group>);
|
||||
|
||||
// Test types
|
||||
static_assert(cuda::std::is_same_v<Unit, typename Group::unit_type>);
|
||||
static_assert(cuda::std::is_same_v<Level, typename Group::level_type>);
|
||||
|
||||
// Test that the group can be queried for it's hierarchy.
|
||||
{
|
||||
decltype(auto) hierarchy = cuda::std::as_const(group).hierarchy();
|
||||
static_assert(cuda::std::is_same_v<decltype(hierarchy), const Hierarchy&>);
|
||||
}
|
||||
|
||||
// Test that the group can be synchronized using .sync() method.
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<void, decltype(group.sync())>);
|
||||
static_assert(noexcept(group.sync()));
|
||||
|
||||
// .sync() method must support calls from different branches. Add some dummy work to make sure the branches are not
|
||||
// collided.
|
||||
cuda::atomic_ref<unsigned, cuda::thread_scope_device> atomic{global_var};
|
||||
if ((threadIdx.x + threadIdx.y + threadIdx.z) % 2 == 0)
|
||||
{
|
||||
atomic++;
|
||||
group.sync();
|
||||
atomic--;
|
||||
}
|
||||
else
|
||||
{
|
||||
atomic--;
|
||||
group.sync();
|
||||
atomic++;
|
||||
}
|
||||
}
|
||||
|
||||
// Test that the group can be synchronized using .sync_aligned() method.
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<void, decltype(group.sync_aligned())>);
|
||||
static_assert(noexcept(group.sync_aligned()));
|
||||
|
||||
// .sync_aligned() method must be called by all threads in the group uniformly in one place.
|
||||
group.sync_aligned();
|
||||
}
|
||||
}
|
||||
|
||||
template <class ParentGroup, cuda::std::size_t N, class Synchronizer>
|
||||
__device__ void
|
||||
test_queries(const cudax::group<cuda::thread_level, ParentGroup, cudax::group_by<N>, Synchronizer>& group)
|
||||
{
|
||||
// todo(dabayer): These queries end up in `error: expression must have a constant value`, when group is taken by
|
||||
// reference. Can we find a solution that works without copying the group?
|
||||
// static_assert(cuda::gpu_thread.static_count(group) == N);
|
||||
|
||||
using Group = cuda::std::remove_cvref_t<decltype(group)>;
|
||||
using Level = typename Group::level_type;
|
||||
|
||||
const auto count_ref = group.__mapping_result().unit_count();
|
||||
const auto rank_ref = cuda::gpu_thread.rank(Level{}, group.hierarchy()) % count_ref;
|
||||
|
||||
REQUIRE(cuda::gpu_thread.count(group) == count_ref);
|
||||
REQUIRE(cuda::gpu_thread.rank(group) == rank_ref);
|
||||
REQUIRE(cuda::gpu_thread.is_root_rank(group) == (rank_ref == 0));
|
||||
REQUIRE(cuda::gpu_thread.is_part_of(group));
|
||||
|
||||
auto group_count_ref = group.__mapping_result().group_count();
|
||||
auto group_rank_ref = group.__mapping_result().group_rank();
|
||||
|
||||
if constexpr (!cuda::std::is_same_v<Level, cuda::grid_level>)
|
||||
{
|
||||
group_count_ref *= Level{}.count(cuda::grid, group.hierarchy());
|
||||
group_rank_ref += group.__mapping_result().group_count() * Level{}.rank(cuda::grid, group.hierarchy());
|
||||
}
|
||||
|
||||
REQUIRE(group.count(cuda::grid) == group_count_ref);
|
||||
REQUIRE(group.rank(cuda::grid) == group_rank_ref);
|
||||
}
|
||||
|
||||
template <cuda::std::size_t N, class Unit, class Level, class Config>
|
||||
__device__ void test_group_by_group(Unit unit, Level level, Config config)
|
||||
{
|
||||
constexpr cuda::std::size_t nbarriers = unit.static_count(level, config) / N;
|
||||
|
||||
auto parent_group = cudax::make_this_group(level, config);
|
||||
|
||||
{
|
||||
auto& barriers = get_barriers<nbarriers, 0>(level);
|
||||
|
||||
cudax::group_by<N> mapping{};
|
||||
cudax::barrier_synchronizer synchronizer{barriers};
|
||||
cudax::group group{unit, parent_group, mapping, synchronizer};
|
||||
|
||||
static_assert(
|
||||
cuda::std::is_same_v<cudax::group<Unit, decltype(parent_group), decltype(mapping), decltype(synchronizer)>,
|
||||
decltype(group)>);
|
||||
test_common_properties<Unit, Level>(config.hierarchy(), group);
|
||||
test_queries(group);
|
||||
group.sync();
|
||||
}
|
||||
{
|
||||
auto& barriers = get_barriers<nbarriers, 1>(level);
|
||||
|
||||
cudax::group_by mapping{N};
|
||||
cudax::barrier_synchronizer synchronizer{barriers};
|
||||
cudax::group group{unit, parent_group, mapping, synchronizer};
|
||||
|
||||
static_assert(
|
||||
cuda::std::is_same_v<cudax::group<Unit, decltype(parent_group), decltype(mapping), decltype(synchronizer)>,
|
||||
decltype(group)>);
|
||||
test_common_properties<Unit, Level>(config.hierarchy(), group);
|
||||
test_queries(group);
|
||||
group.sync();
|
||||
}
|
||||
}
|
||||
|
||||
template <class Unit, class Level, class Config>
|
||||
__device__ void test_group_by_group(const Unit& unit, const Level& level, const Config& config)
|
||||
{
|
||||
// powers of 2
|
||||
test_group_by_group<1>(unit, level, config);
|
||||
test_group_by_group<4>(unit, level, config);
|
||||
test_group_by_group<16>(unit, level, config);
|
||||
test_group_by_group<32>(unit, level, config);
|
||||
|
||||
if constexpr (!cuda::std::is_same_v<Level, cuda::warp_level>)
|
||||
{
|
||||
test_group_by_group<64>(unit, level, config);
|
||||
test_group_by_group<128>(unit, level, config);
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
// todo(dabayer): Investigate why enabling warp leads to launch failure and cluster deadlocks.
|
||||
// test_group_by_group(cuda::gpu_thread, cuda::warp, config);
|
||||
test_group_by_group(cuda::gpu_thread, cuda::block, config);
|
||||
// test_group_by_group(cuda::gpu_thread, cuda::cluster, config);
|
||||
test_group_by_group(cuda::gpu_thread, cuda::grid, config);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("Group", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
const auto config = cuda::make_config(cuda::grid_dims<2>(), cuda::block_dims<128>(), cuda::cooperative_launch{});
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
|
||||
if (cuda::device_attributes::compute_capability(device) >= cuda::compute_capability{90})
|
||||
{
|
||||
const auto config_cluster = cuda::make_config(
|
||||
cuda::grid_dims<2>(), cuda::cluster_dims<3>(), cuda::block_dims<128>(), cuda::cooperative_launch{});
|
||||
cuda::launch(stream, config_cluster, TestKernel{});
|
||||
}
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,290 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/atomic>
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
__device__ cuda::std::size_t invoke_count;
|
||||
__device__ int global_value = 1;
|
||||
|
||||
__device__ void update_invoke_count() noexcept
|
||||
{
|
||||
cuda::atomic_ref<cuda::std::size_t, cuda::thread_scope_device>
|
||||
{
|
||||
invoke_count
|
||||
}
|
||||
++;
|
||||
}
|
||||
|
||||
template <class Group>
|
||||
__device__ void check_and_reset_invoke_count(const Group& group)
|
||||
{
|
||||
group.sync_aligned();
|
||||
__threadfence();
|
||||
|
||||
REQUIRE(cuda::atomic_ref<cuda::std::size_t, cuda::thread_scope_device>{invoke_count} == 1);
|
||||
__threadfence();
|
||||
|
||||
if (cuda::gpu_thread.is_root_rank(group))
|
||||
{
|
||||
cuda::atomic_ref<cuda::std::size_t, cuda::thread_scope_device>{invoke_count} = 0;
|
||||
}
|
||||
__threadfence();
|
||||
group.sync_aligned();
|
||||
}
|
||||
|
||||
template <class Group>
|
||||
__device__ void test_invoke_one(const Group& group)
|
||||
{
|
||||
// We need only 1 group for these tests.
|
||||
if (group.rank(cuda::grid) > 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// invoke_one callable with void return type
|
||||
{
|
||||
auto callable = []() {
|
||||
update_invoke_count();
|
||||
};
|
||||
|
||||
static_assert(cuda::std::is_same_v<void, decltype(cudax::invoke_one(group, callable))>);
|
||||
static_assert(!noexcept(cudax::invoke_one(group, callable)));
|
||||
|
||||
cudax::invoke_one(group, callable);
|
||||
|
||||
check_and_reset_invoke_count(group);
|
||||
}
|
||||
|
||||
// invoke_one nothrow callable with void return type
|
||||
{
|
||||
auto callable = []() noexcept {
|
||||
update_invoke_count();
|
||||
};
|
||||
|
||||
static_assert(cuda::std::is_same_v<void, decltype(cudax::invoke_one(group, callable))>);
|
||||
static_assert(noexcept(cudax::invoke_one(group, callable)));
|
||||
|
||||
cudax::invoke_one(group, callable);
|
||||
|
||||
check_and_reset_invoke_count(group);
|
||||
}
|
||||
|
||||
// invoke_one callable with value return type
|
||||
{
|
||||
auto callable = []() -> int {
|
||||
update_invoke_count();
|
||||
return 1;
|
||||
};
|
||||
|
||||
static_assert(cuda::std::is_same_v<cuda::std::optional<int>, decltype(cudax::invoke_one(group, callable))>);
|
||||
static_assert(!noexcept(cudax::invoke_one(group, callable)));
|
||||
|
||||
const auto ret = cudax::invoke_one(group, callable);
|
||||
REQUIRE(ret.has_value() == cudax::__elect_one(group));
|
||||
if (ret.has_value())
|
||||
{
|
||||
REQUIRE(ret == 1);
|
||||
}
|
||||
|
||||
check_and_reset_invoke_count(group);
|
||||
}
|
||||
|
||||
// invoke_one nothrow callable with value return type
|
||||
{
|
||||
auto callable = []() noexcept -> int {
|
||||
update_invoke_count();
|
||||
return 1;
|
||||
};
|
||||
|
||||
static_assert(cuda::std::is_same_v<cuda::std::optional<int>, decltype(cudax::invoke_one(group, callable))>);
|
||||
static_assert(noexcept(cudax::invoke_one(group, callable)));
|
||||
|
||||
const auto ret = cudax::invoke_one(group, callable);
|
||||
REQUIRE(ret.has_value() == cudax::__elect_one(group));
|
||||
if (ret.has_value())
|
||||
{
|
||||
REQUIRE(ret == 1);
|
||||
}
|
||||
|
||||
check_and_reset_invoke_count(group);
|
||||
}
|
||||
|
||||
// invoke_one callable with l-value reference return type
|
||||
{
|
||||
auto callable = []() -> int& {
|
||||
update_invoke_count();
|
||||
return global_value;
|
||||
};
|
||||
|
||||
static_assert(cuda::std::is_same_v<cuda::std::optional<int&>, decltype(cudax::invoke_one(group, callable))>);
|
||||
static_assert(!noexcept(cudax::invoke_one(group, callable)));
|
||||
|
||||
const auto ret = cudax::invoke_one(group, callable);
|
||||
REQUIRE(ret.has_value() == cudax::__elect_one(group));
|
||||
if (ret.has_value())
|
||||
{
|
||||
REQUIRE(ret == 1);
|
||||
REQUIRE(&ret.value() == &global_value);
|
||||
}
|
||||
|
||||
check_and_reset_invoke_count(group);
|
||||
}
|
||||
|
||||
// invoke_one callable with l-value reference return type
|
||||
{
|
||||
auto callable = []() noexcept -> int& {
|
||||
update_invoke_count();
|
||||
return global_value;
|
||||
};
|
||||
|
||||
static_assert(cuda::std::is_same_v<cuda::std::optional<int&>, decltype(cudax::invoke_one(group, callable))>);
|
||||
static_assert(noexcept(cudax::invoke_one(group, callable)));
|
||||
|
||||
const auto ret = cudax::invoke_one(group, callable);
|
||||
REQUIRE(ret.has_value() == cudax::__elect_one(group));
|
||||
if (ret.has_value())
|
||||
{
|
||||
REQUIRE(ret == 1);
|
||||
REQUIRE(&ret.value() == &global_value);
|
||||
}
|
||||
|
||||
check_and_reset_invoke_count(group);
|
||||
}
|
||||
|
||||
// invoke_one callable with r-value reference return type
|
||||
{
|
||||
auto callable = []() -> int&& {
|
||||
update_invoke_count();
|
||||
return cuda::std::move(global_value);
|
||||
};
|
||||
|
||||
static_assert(cuda::std::is_same_v<cuda::std::optional<int>, decltype(cudax::invoke_one(group, callable))>);
|
||||
static_assert(!noexcept(cudax::invoke_one(group, callable)));
|
||||
|
||||
const auto ret = cudax::invoke_one(group, callable);
|
||||
REQUIRE(ret.has_value() == cudax::__elect_one(group));
|
||||
if (ret.has_value())
|
||||
{
|
||||
REQUIRE(ret == 1);
|
||||
}
|
||||
|
||||
check_and_reset_invoke_count(group);
|
||||
}
|
||||
|
||||
// invoke_one nothrow callable with r-value reference return type
|
||||
{
|
||||
auto callable = []() noexcept -> int&& {
|
||||
update_invoke_count();
|
||||
return cuda::std::move(global_value);
|
||||
};
|
||||
|
||||
static_assert(cuda::std::is_same_v<cuda::std::optional<int>, decltype(cudax::invoke_one(group, callable))>);
|
||||
static_assert(noexcept(cudax::invoke_one(group, callable)));
|
||||
|
||||
const auto ret = cudax::invoke_one(group, callable);
|
||||
REQUIRE(ret.has_value() == cudax::__elect_one(group));
|
||||
if (ret.has_value())
|
||||
{
|
||||
REQUIRE(ret == 1);
|
||||
}
|
||||
|
||||
check_and_reset_invoke_count(group);
|
||||
}
|
||||
|
||||
// Check that invoke_one correctly forwards the arguments for invocable with void return type.
|
||||
{
|
||||
auto callable = [](auto&& arg1, auto&& arg2) -> void {
|
||||
static_assert(cuda::std::is_same_v<int&, decltype(arg1)>);
|
||||
static_assert(cuda::std::is_same_v<unsigned&&, decltype(arg2)>);
|
||||
|
||||
REQUIRE(arg1 == 2);
|
||||
REQUIRE(arg2 == 20u);
|
||||
};
|
||||
|
||||
int arg1{2};
|
||||
unsigned arg2{20};
|
||||
cudax::invoke_one(group, callable, arg1, cuda::std::move(arg2));
|
||||
}
|
||||
|
||||
// Check that invoke_one correctly forwards the arguments for invocable with non-void return type.
|
||||
{
|
||||
auto callable = [](auto&& arg1, auto&& arg2) -> int {
|
||||
static_assert(cuda::std::is_same_v<int&, decltype(arg1)>);
|
||||
static_assert(cuda::std::is_same_v<unsigned&&, decltype(arg2)>);
|
||||
|
||||
REQUIRE(arg1 == 2);
|
||||
REQUIRE(arg2 == 20u);
|
||||
|
||||
return 1;
|
||||
};
|
||||
|
||||
int arg1{2};
|
||||
unsigned arg2{20};
|
||||
const auto ret = cudax::invoke_one(group, callable, arg1, cuda::std::move(arg2));
|
||||
|
||||
REQUIRE(ret.has_value() == cudax::__elect_one(group));
|
||||
if (ret.has_value())
|
||||
{
|
||||
REQUIRE(ret == 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
// Test this groups.
|
||||
test_invoke_one(cudax::this_thread{config});
|
||||
test_invoke_one(cudax::this_warp{config});
|
||||
test_invoke_one(cudax::this_block{config});
|
||||
test_invoke_one(cudax::this_cluster{config});
|
||||
|
||||
// Test custom groups.
|
||||
{
|
||||
cudax::group group{cuda::gpu_thread, cudax::this_warp{config}, cudax::group_by<4>{}, cudax::lane_synchronizer{}};
|
||||
test_invoke_one(group);
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("Invoke one", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
const auto config = cuda::make_config(cuda::grid_dims<2>(), cuda::block_dims<128>(), cuda::cooperative_launch{});
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
|
||||
if (cuda::device_attributes::compute_capability(device) >= cuda::compute_capability{90})
|
||||
{
|
||||
const auto config_cluster = cuda::make_config(
|
||||
cuda::grid_dims<2>(), cuda::cluster_dims<3>(), cuda::block_dims<128>(), cuda::cooperative_launch{});
|
||||
cuda::launch(stream, config_cluster, TestKernel{});
|
||||
}
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
template <template <class> class GroupTempl, class Level, class Config>
|
||||
__device__ void test_make_this_group(const Level& level, const Config& config)
|
||||
{
|
||||
// Test default construction.
|
||||
{
|
||||
static_assert(
|
||||
cuda::std::is_same_v<GroupTempl<cudax::__implicit_hierarchy_t>, decltype(cudax::make_this_group(level))>);
|
||||
static_assert(noexcept(cudax::make_this_group(level)));
|
||||
|
||||
auto group = cudax::make_this_group(level);
|
||||
group.sync();
|
||||
}
|
||||
|
||||
// Test construction from hierarchy-like.
|
||||
{
|
||||
using Hierarchy = typename Config::hierarchy_type;
|
||||
|
||||
static_assert(cuda::std::is_same_v<GroupTempl<Hierarchy>, decltype(cudax::make_this_group(level, config))>);
|
||||
static_assert(noexcept(cudax::make_this_group(level, config)));
|
||||
|
||||
auto group = cudax::make_this_group(level, config);
|
||||
group.sync();
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_make_this_group<cudax::this_thread>(cuda::gpu_thread, config);
|
||||
test_make_this_group<cudax::this_warp>(cuda::warp, config);
|
||||
test_make_this_group<cudax::this_block>(cuda::block, config);
|
||||
test_make_this_group<cudax::this_cluster>(cuda::cluster, config);
|
||||
test_make_this_group<cudax::this_grid>(cuda::grid, config);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("Make this group", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
const auto config = cuda::make_config(cuda::grid_dims<2>(), cuda::block_dims<128>(), cuda::cooperative_launch{});
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
|
||||
if (cuda::device_attributes::compute_capability(device) >= cuda::compute_capability{90})
|
||||
{
|
||||
const auto config_cluster = cuda::make_config(
|
||||
cuda::grid_dims<2>(), cuda::cluster_dims<3>(), cuda::block_dims<128>(), cuda::cooperative_launch{});
|
||||
cuda::launch(stream, config_cluster, TestKernel{});
|
||||
}
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,209 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
struct AlwaysTruePredFn
|
||||
{
|
||||
template <class MappingResult>
|
||||
__device__ bool operator()(MappingResult mapping_result)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
struct AlwaysFalsePredFn
|
||||
{
|
||||
template <class MappingResult>
|
||||
__device__ bool operator()(MappingResult mapping_result) noexcept
|
||||
{
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
struct IsEvenPredFn
|
||||
{
|
||||
template <class MappingResult>
|
||||
__device__ bool operator()(MappingResult mapping_result)
|
||||
{
|
||||
return mapping_result.unit_rank() % 2 == 0;
|
||||
}
|
||||
};
|
||||
|
||||
template <class Config>
|
||||
__device__ void test_binary_partition(Config config)
|
||||
{
|
||||
// Always true predicate.
|
||||
{
|
||||
using Pred = AlwaysTruePredFn;
|
||||
using Mapping = cudax::binary_partition<Pred>;
|
||||
|
||||
// Test constructor from pred_fn.
|
||||
{
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<Mapping, Pred>);
|
||||
cudax::binary_partition mapping{Pred{}};
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
!noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping{Pred{}};
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
static_assert(Result::static_group_count() == 2);
|
||||
REQUIRE(result.group_count() == 2);
|
||||
REQUIRE(result.group_rank() == 1);
|
||||
|
||||
static_assert(Result::static_unit_count() == cuda::std::dynamic_extent);
|
||||
REQUIRE(result.unit_count() == cuda::gpu_thread.count(cuda::warp));
|
||||
REQUIRE(result.unit_rank() == cuda::gpu_thread.rank(cuda::warp));
|
||||
|
||||
REQUIRE(result.lane_mask() == cuda::device::lane_mask::all());
|
||||
|
||||
REQUIRE(result.is_valid());
|
||||
static_assert(Result::is_always_exhaustive());
|
||||
static_assert(!Result::is_always_contiguous());
|
||||
}
|
||||
}
|
||||
|
||||
// Always false predicate.
|
||||
{
|
||||
using Pred = AlwaysFalsePredFn;
|
||||
using Mapping = cudax::binary_partition<Pred>;
|
||||
|
||||
// Test constructor from pred_fn.
|
||||
{
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<Mapping, Pred>);
|
||||
cudax::binary_partition mapping{Pred{}};
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping{Pred{}};
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
static_assert(Result::static_group_count() == 2);
|
||||
REQUIRE(result.group_count() == 2);
|
||||
REQUIRE(result.group_rank() == 0);
|
||||
|
||||
static_assert(Result::static_unit_count() == cuda::std::dynamic_extent);
|
||||
REQUIRE(result.unit_count() == cuda::gpu_thread.count(cuda::warp));
|
||||
REQUIRE(result.unit_rank() == cuda::gpu_thread.rank(cuda::warp));
|
||||
|
||||
REQUIRE(result.lane_mask() == cuda::device::lane_mask::all());
|
||||
|
||||
REQUIRE(result.is_valid());
|
||||
static_assert(Result::is_always_exhaustive());
|
||||
static_assert(!Result::is_always_contiguous());
|
||||
}
|
||||
}
|
||||
|
||||
// True for even ranks predicate.
|
||||
{
|
||||
using Pred = IsEvenPredFn;
|
||||
using Mapping = cudax::binary_partition<Pred>;
|
||||
|
||||
// Test constructor from pred_fn.
|
||||
{
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<Mapping, Pred>);
|
||||
cudax::binary_partition mapping{Pred{}};
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
!noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping{Pred{}};
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
static_assert(Result::static_group_count() == 2);
|
||||
REQUIRE(result.group_count() == 2);
|
||||
REQUIRE(result.group_rank() == (cuda::gpu_thread.rank(cuda::warp) % 2 == 0));
|
||||
|
||||
static_assert(Result::static_unit_count() == cuda::std::dynamic_extent);
|
||||
REQUIRE(result.unit_count() == cuda::gpu_thread.count(cuda::warp) / 2);
|
||||
REQUIRE(result.unit_rank() == cuda::gpu_thread.rank(cuda::warp) / 2);
|
||||
|
||||
const auto lane_mask_ref =
|
||||
(cuda::gpu_thread.rank(cuda::warp) % 2 == 0)
|
||||
? cuda::device::lane_mask{0x5555'5555u}
|
||||
: cuda::device::lane_mask(0xaaaa'aaaau);
|
||||
REQUIRE(result.lane_mask() == lane_mask_ref);
|
||||
|
||||
REQUIRE(result.is_valid());
|
||||
static_assert(Result::is_always_exhaustive());
|
||||
static_assert(!Result::is_always_contiguous());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_binary_partition(config);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("Binary partition mapping", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<8, 4>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims(dim3{8, 4}));
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,196 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/numeric>
|
||||
#include <cuda/std/tuple>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
#include <cuda/warp>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
template <class Mapping1, class Mapping2, class Config>
|
||||
__device__ void test_composite_mapping(const Mapping1& mapping1, const Mapping2& mapping2, Config config)
|
||||
{
|
||||
using Mapping = cudax::composite_mapping<Mapping1, Mapping2>;
|
||||
|
||||
// Test construction from 2 mappings.
|
||||
{
|
||||
cudax::composite_mapping mapping{mapping1, mapping2};
|
||||
static_assert(cuda::std::is_same_v<decltype(mapping), Mapping>);
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<Mapping, Mapping1, Mapping2>
|
||||
== (cuda::std::is_nothrow_copy_constructible_v<Mapping1>
|
||||
&& cuda::std::is_nothrow_copy_constructible_v<Mapping2>) );
|
||||
}
|
||||
|
||||
// Test get().
|
||||
{
|
||||
const cudax::composite_mapping mapping{mapping1, mapping2};
|
||||
static_assert(cuda::std::is_same_v<decltype(mapping.get()), const cuda::std::tuple<Mapping1, Mapping2>&>);
|
||||
static_assert(noexcept(mapping.get()));
|
||||
|
||||
const auto& mapping1_ref = cuda::std::get<0>(mapping.get());
|
||||
CHECK(mapping1_ref.unit_count() == 4);
|
||||
|
||||
const auto& mapping2_ref = cuda::std::get<1>(mapping.get());
|
||||
CHECK(mapping2_ref.unit_count(0) == 1);
|
||||
CHECK(mapping2_ref.unit_count(1) == 3);
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
const cudax::composite_mapping mapping{mapping1, mapping2};
|
||||
|
||||
static_assert(
|
||||
cudax::__group_mapping_result<decltype(mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
const auto rank_in_warp = cuda::gpu_thread.rank_as<unsigned>(parent_group);
|
||||
|
||||
if constexpr (Mapping1::static_unit_count() != cuda::std::dynamic_extent
|
||||
&& Mapping2::static_group_count() != cuda::std::dynamic_extent)
|
||||
{
|
||||
static_assert(Result::static_group_count() == 16);
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(Result::static_group_count() == cuda::std::dynamic_extent);
|
||||
}
|
||||
CHECK(result.group_count() == 16);
|
||||
CHECK(result.group_rank() == (rank_in_warp / 4 * 2 + (rank_in_warp % 4 > 0)));
|
||||
|
||||
static_assert(Result::static_unit_count() == cuda::std::dynamic_extent);
|
||||
CHECK(result.unit_count() == ((rank_in_warp % 4 > 0) ? 3 : 1));
|
||||
CHECK(result.unit_rank() == ((rank_in_warp % 4 > 0) ? (rank_in_warp % 4 - 1) : 0));
|
||||
|
||||
const auto lane_mask_ref = ((rank_in_warp % 4 > 0) ? 0b1110u : 0b0001u) << ((rank_in_warp / 4) * 4);
|
||||
CHECK(result.lane_mask() == cuda::device::lane_mask{lane_mask_ref});
|
||||
|
||||
CHECK(result.is_valid());
|
||||
static_assert(Result::is_always_exhaustive());
|
||||
static_assert(Result::is_always_contiguous());
|
||||
}
|
||||
|
||||
// Test operator|.
|
||||
{
|
||||
auto mapping = mapping1 | mapping2;
|
||||
|
||||
static_assert(cuda::std::is_same_v<Mapping, decltype(mapping)>);
|
||||
static_assert(noexcept(mapping1 | mapping2));
|
||||
|
||||
const auto& mapping1_ref = cuda::std::get<0>(mapping.get());
|
||||
CHECK(mapping1_ref.unit_count() == 4);
|
||||
|
||||
const auto& mapping2_ref = cuda::std::get<1>(mapping.get());
|
||||
CHECK(mapping2_ref.unit_count(0) == 1);
|
||||
CHECK(mapping2_ref.unit_count(1) == 3);
|
||||
}
|
||||
{
|
||||
auto mapping = cudax::composite_mapping{mapping1} | mapping2;
|
||||
|
||||
static_assert(cuda::std::is_same_v<Mapping, decltype(mapping)>);
|
||||
static_assert(noexcept(cudax::composite_mapping{mapping1} | mapping2));
|
||||
|
||||
const auto& mapping1_ref = cuda::std::get<0>(mapping.get());
|
||||
CHECK(mapping1_ref.unit_count() == 4);
|
||||
|
||||
const auto& mapping2_ref = cuda::std::get<1>(mapping.get());
|
||||
CHECK(mapping2_ref.unit_count(0) == 1);
|
||||
CHECK(mapping2_ref.unit_count(1) == 3);
|
||||
}
|
||||
{
|
||||
auto mapping = mapping1 | cudax::composite_mapping{mapping2};
|
||||
|
||||
static_assert(cuda::std::is_same_v<Mapping, decltype(mapping)>);
|
||||
static_assert(noexcept(mapping1 | cudax::composite_mapping{mapping2}));
|
||||
|
||||
const auto& mapping1_ref = cuda::std::get<0>(mapping.get());
|
||||
CHECK(mapping1_ref.unit_count() == 4);
|
||||
|
||||
const auto& mapping2_ref = cuda::std::get<1>(mapping.get());
|
||||
CHECK(mapping2_ref.unit_count(0) == 1);
|
||||
CHECK(mapping2_ref.unit_count(1) == 3);
|
||||
}
|
||||
{
|
||||
auto mapping = cudax::composite_mapping{mapping1} | cudax::composite_mapping{mapping2};
|
||||
|
||||
static_assert(cuda::std::is_same_v<Mapping, decltype(mapping)>);
|
||||
static_assert(noexcept(cudax::composite_mapping{mapping1} | cudax::composite_mapping{mapping2}));
|
||||
|
||||
const auto& mapping1_ref = cuda::std::get<0>(mapping.get());
|
||||
CHECK(mapping1_ref.unit_count() == 4);
|
||||
|
||||
const auto& mapping2_ref = cuda::std::get<1>(mapping.get());
|
||||
CHECK(mapping2_ref.unit_count(0) == 1);
|
||||
CHECK(mapping2_ref.unit_count(1) == 3);
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
{
|
||||
const cudax::group_by<4> mapping1{};
|
||||
const cudax::group_as mapping2{cuda::std::integer_sequence<cuda::std::size_t, 1, 3>{}};
|
||||
test_composite_mapping(mapping1, mapping2, config);
|
||||
}
|
||||
{
|
||||
const cudax::group_by mapping1{4};
|
||||
const cudax::group_as mapping2{cuda::std::integer_sequence<cuda::std::size_t, 1, 3>{}};
|
||||
test_composite_mapping(mapping1, mapping2, config);
|
||||
}
|
||||
{
|
||||
const cudax::group_by<4> mapping1{};
|
||||
constexpr unsigned counts2[]{1, 3};
|
||||
const cudax::group_as mapping2{counts2};
|
||||
test_composite_mapping(mapping1, mapping2, config);
|
||||
}
|
||||
{
|
||||
const cudax::group_by mapping1{4};
|
||||
constexpr unsigned counts2[]{1, 3};
|
||||
const cudax::group_as mapping2{counts2};
|
||||
test_composite_mapping(mapping1, mapping2, config);
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("Composite mapping", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<8, 4>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims(dim3{8, 4}));
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,528 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/numeric>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
#include <cuda/warp>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
template <cuda::std::size_t... Ns, class Config>
|
||||
__device__ void test_group_as(Config config)
|
||||
{
|
||||
using NsSeq = cuda::std::integer_sequence<cuda::std::size_t, Ns...>;
|
||||
constexpr unsigned ns[]{static_cast<unsigned>(Ns)...};
|
||||
constexpr cuda::std::size_t ngroups = sizeof...(Ns);
|
||||
|
||||
cuda::std::size_t group_starts[ngroups];
|
||||
cuda::std::exclusive_scan(cuda::std::begin(ns), cuda::std::end(ns), group_starts, cuda::std::size_t{});
|
||||
|
||||
// Test static Ns.
|
||||
{
|
||||
using Mapping = cudax::group_as<cudax::__group_as_static_tag<Ns...>, true>;
|
||||
|
||||
// Test default constructor.
|
||||
{
|
||||
static_assert(cuda::std::is_trivially_default_constructible_v<Mapping>);
|
||||
static_assert(cuda::std::is_empty_v<Mapping>);
|
||||
|
||||
Mapping mapping;
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(mapping.unit_count(i) == ns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Test the mapping is constructible from the Ns sequence.
|
||||
{
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<Mapping, NsSeq>);
|
||||
|
||||
cudax::group_as mapping{NsSeq{}};
|
||||
static_assert(cuda::std::is_same_v<decltype(mapping), Mapping>);
|
||||
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(mapping.unit_count(i) == ns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Test the mapping is not constructible from Ns sequence and non_exhaustive_t.
|
||||
static_assert(!cuda::std::is_constructible_v<Mapping, NsSeq, cudax::non_exhaustive_t>);
|
||||
|
||||
// Test static_group_count().
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_group_count())>);
|
||||
static_assert(noexcept(Mapping::static_group_count()));
|
||||
static_assert(Mapping::static_group_count() == ngroups);
|
||||
|
||||
// Test static_unit_count().
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_unit_count(cuda::std::size_t{}))>);
|
||||
static_assert(noexcept(Mapping::static_unit_count(cuda::std::size_t{})));
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(Mapping::static_unit_count(i) == ns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Test is_always_exhaustive().
|
||||
static_assert(cuda::std::is_same_v<bool, decltype(Mapping::is_always_exhaustive())>);
|
||||
static_assert(noexcept(Mapping::is_always_exhaustive()));
|
||||
static_assert(Mapping::is_always_exhaustive());
|
||||
|
||||
// Test unit_count().
|
||||
{
|
||||
static_assert(
|
||||
cuda::std::is_same_v<unsigned, decltype(cuda::std::declval<const Mapping>().unit_count(cuda::std::size_t{}))>);
|
||||
static_assert(noexcept(cuda::std::declval<const Mapping>().unit_count(cuda::std::size_t{})));
|
||||
|
||||
const Mapping mapping;
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(mapping.unit_count(i) == ns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping;
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
const auto rank_in_warp = cuda::gpu_thread.rank(parent_group);
|
||||
|
||||
unsigned group_rank_ref = ngroups - 1;
|
||||
unsigned rank_ref = rank_in_warp - group_starts[group_rank_ref];
|
||||
for (unsigned i = 1; i < ngroups; ++i)
|
||||
{
|
||||
if (rank_in_warp < group_starts[i])
|
||||
{
|
||||
group_rank_ref = i - 1;
|
||||
rank_ref = rank_in_warp - group_starts[i - 1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static_assert(Result::static_group_count() == ngroups);
|
||||
CHECK(result.group_count() == static_cast<unsigned>(ngroups));
|
||||
CHECK(result.group_rank() == group_rank_ref);
|
||||
|
||||
static_assert(Result::static_unit_count() == cuda::std::dynamic_extent);
|
||||
CHECK(result.unit_count() == ns[group_rank_ref]);
|
||||
CHECK(result.unit_rank() == rank_ref);
|
||||
|
||||
const auto lane_mask_ref =
|
||||
(ns[group_rank_ref] < 32) ? ((1u << ns[group_rank_ref]) - 1) << group_starts[group_rank_ref] : ~0;
|
||||
CHECK(result.lane_mask() == cuda::device::lane_mask{lane_mask_ref});
|
||||
|
||||
CHECK(result.is_valid());
|
||||
static_assert(Result::is_always_exhaustive());
|
||||
static_assert(Result::is_always_contiguous());
|
||||
}
|
||||
}
|
||||
|
||||
// Test dynamic Ns.
|
||||
{
|
||||
using Mapping = cudax::group_as<cudax::__group_as_dynamic_tag<ngroups>, true>;
|
||||
|
||||
// Test default constructor.
|
||||
static_assert(!cuda::std::is_default_constructible_v<Mapping>);
|
||||
|
||||
// Test the mapping is constructible from the Ns array.
|
||||
{
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<Mapping, decltype(ns)>);
|
||||
|
||||
cudax::group_as mapping{ns};
|
||||
static_assert(cuda::std::is_same_v<decltype(mapping), Mapping>);
|
||||
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(mapping.unit_count(i) == ns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Test the mapping is not constructible from Ns array and non_exhaustive_t.
|
||||
static_assert(!cuda::std::is_constructible_v<Mapping, decltype(ns), cudax::non_exhaustive_t>);
|
||||
|
||||
// Test static_group_count().
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_group_count())>);
|
||||
static_assert(noexcept(Mapping::static_group_count()));
|
||||
static_assert(Mapping::static_group_count() == ngroups);
|
||||
|
||||
// Test static_unit_count().
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_unit_count(cuda::std::size_t{}))>);
|
||||
static_assert(noexcept(Mapping::static_unit_count(cuda::std::size_t{})));
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(Mapping::static_unit_count(i) == cuda::std::dynamic_extent);
|
||||
}
|
||||
}
|
||||
|
||||
// Test is_always_exhaustive().
|
||||
static_assert(cuda::std::is_same_v<bool, decltype(Mapping::is_always_exhaustive())>);
|
||||
static_assert(noexcept(Mapping::is_always_exhaustive()));
|
||||
static_assert(Mapping::is_always_exhaustive());
|
||||
|
||||
// Test unit_count().
|
||||
{
|
||||
static_assert(
|
||||
cuda::std::is_same_v<unsigned, decltype(cuda::std::declval<const Mapping>().unit_count(cuda::std::size_t{}))>);
|
||||
static_assert(noexcept(cuda::std::declval<const Mapping>().unit_count(cuda::std::size_t{})));
|
||||
|
||||
const Mapping mapping{ns};
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(mapping.unit_count(i) == ns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping{ns};
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
const auto rank_in_warp = cuda::gpu_thread.rank_as<unsigned>(parent_group);
|
||||
|
||||
unsigned group_rank_ref = ngroups - 1;
|
||||
unsigned rank_ref = rank_in_warp - group_starts[group_rank_ref];
|
||||
for (unsigned i = 1; i < ngroups; ++i)
|
||||
{
|
||||
if (rank_in_warp < group_starts[i])
|
||||
{
|
||||
group_rank_ref = i - 1;
|
||||
rank_ref = rank_in_warp - group_starts[i - 1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
static_assert(Result::static_group_count() == ngroups);
|
||||
CHECK(result.group_count() == static_cast<unsigned>(ngroups));
|
||||
CHECK(result.group_rank() == group_rank_ref);
|
||||
|
||||
static_assert(Result::static_unit_count() == cuda::std::dynamic_extent);
|
||||
CHECK(result.unit_count() == ns[group_rank_ref]);
|
||||
CHECK(result.unit_rank() == rank_ref);
|
||||
|
||||
const auto lane_mask_ref =
|
||||
(ns[group_rank_ref] < 32) ? ((1u << ns[group_rank_ref]) - 1) << group_starts[group_rank_ref] : ~0;
|
||||
CHECK(result.lane_mask() == cuda::device::lane_mask{lane_mask_ref});
|
||||
|
||||
CHECK(result.is_valid());
|
||||
static_assert(Result::is_always_exhaustive());
|
||||
static_assert(Result::is_always_contiguous());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <cuda::std::size_t... Ns, class Config>
|
||||
__device__ void test_group_as_non_exhaustive(Config config)
|
||||
{
|
||||
using NsSeq = cuda::std::integer_sequence<cuda::std::size_t, Ns...>;
|
||||
constexpr unsigned ns[]{static_cast<unsigned>(Ns)...};
|
||||
constexpr cuda::std::size_t ngroups = sizeof...(Ns);
|
||||
|
||||
cuda::std::size_t group_starts[ngroups];
|
||||
cuda::std::exclusive_scan(cuda::std::begin(ns), cuda::std::end(ns), group_starts, cuda::std::size_t{});
|
||||
|
||||
const auto ns_sum = cuda::std::accumulate(cuda::std::begin(ns), cuda::std::end(ns), 0u);
|
||||
|
||||
// Test static Ns.
|
||||
{
|
||||
using Mapping = cudax::group_as<cudax::__group_as_static_tag<Ns...>, false>;
|
||||
|
||||
// Test default constructor.
|
||||
{
|
||||
static_assert(cuda::std::is_trivially_default_constructible_v<Mapping>);
|
||||
static_assert(cuda::std::is_empty_v<Mapping>);
|
||||
|
||||
Mapping mapping;
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(mapping.unit_count(i) == ns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Test the mapping is not constructible from the Ns sequence.
|
||||
static_assert(!cuda::std::is_constructible_v<Mapping, NsSeq>);
|
||||
|
||||
// Test the mapping is constructible from Ns sequence and non_exhaustive_t.
|
||||
{
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<Mapping, NsSeq, cudax::non_exhaustive_t>);
|
||||
|
||||
cudax::group_as mapping{NsSeq{}, cudax::non_exhaustive};
|
||||
static_assert(cuda::std::is_same_v<decltype(mapping), Mapping>);
|
||||
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(mapping.unit_count(i) == ns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Test static_group_count().
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_group_count())>);
|
||||
static_assert(noexcept(Mapping::static_group_count()));
|
||||
static_assert(Mapping::static_group_count() == ngroups);
|
||||
|
||||
// Test static_unit_count().
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_unit_count(cuda::std::size_t{}))>);
|
||||
static_assert(noexcept(Mapping::static_unit_count(cuda::std::size_t{})));
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(Mapping::static_unit_count(i) == ns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Test is_always_exhaustive().
|
||||
static_assert(cuda::std::is_same_v<bool, decltype(Mapping::is_always_exhaustive())>);
|
||||
static_assert(noexcept(Mapping::is_always_exhaustive()));
|
||||
static_assert(!Mapping::is_always_exhaustive());
|
||||
|
||||
// Test unit_count().
|
||||
{
|
||||
static_assert(
|
||||
cuda::std::is_same_v<unsigned, decltype(cuda::std::declval<const Mapping>().unit_count(cuda::std::size_t{}))>);
|
||||
static_assert(noexcept(cuda::std::declval<const Mapping>().unit_count(cuda::std::size_t{})));
|
||||
|
||||
const Mapping mapping;
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(mapping.unit_count(i) == ns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping;
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
const auto rank_in_warp = cuda::gpu_thread.rank(parent_group);
|
||||
const auto is_valid_ref = (rank_in_warp < ns_sum);
|
||||
|
||||
static_assert(Result::static_group_count() == ngroups);
|
||||
static_assert(Result::static_unit_count() == cuda::std::dynamic_extent);
|
||||
static_assert(!Result::is_always_exhaustive());
|
||||
static_assert(Result::is_always_contiguous());
|
||||
|
||||
CHECK(result.group_count() == static_cast<unsigned>(ngroups));
|
||||
CHECK(result.is_valid() == is_valid_ref);
|
||||
|
||||
if (is_valid_ref)
|
||||
{
|
||||
unsigned group_rank_ref = ngroups - 1;
|
||||
unsigned rank_ref = rank_in_warp - group_starts[group_rank_ref];
|
||||
for (unsigned i = 1; i < ngroups; ++i)
|
||||
{
|
||||
if (rank_in_warp < group_starts[i])
|
||||
{
|
||||
group_rank_ref = i - 1;
|
||||
rank_ref = rank_in_warp - group_starts[i - 1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
CHECK(result.group_rank() == group_rank_ref);
|
||||
|
||||
CHECK(result.unit_count() == ns[group_rank_ref]);
|
||||
CHECK(result.unit_rank() == rank_ref);
|
||||
|
||||
const auto lane_mask_ref =
|
||||
(ns[group_rank_ref] < 32) ? ((1u << ns[group_rank_ref]) - 1) << group_starts[group_rank_ref] : ~0;
|
||||
CHECK(result.lane_mask() == cuda::device::lane_mask{lane_mask_ref});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test dynamic Ns.
|
||||
{
|
||||
using Mapping = cudax::group_as<cudax::__group_as_dynamic_tag<ngroups>, false>;
|
||||
|
||||
// Test default constructor.
|
||||
static_assert(!cuda::std::is_default_constructible_v<Mapping>);
|
||||
|
||||
// Test the mapping is not constructible from the Ns array.
|
||||
static_assert(!cuda::std::is_constructible_v<Mapping, decltype(ns)>);
|
||||
|
||||
// Test the mapping is constructible from Ns array and non_exhaustive_t.
|
||||
{
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<Mapping, decltype(ns), cudax::non_exhaustive_t>);
|
||||
|
||||
cudax::group_as mapping{ns, cudax::non_exhaustive};
|
||||
static_assert(cuda::std::is_same_v<decltype(mapping), Mapping>);
|
||||
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(mapping.unit_count(i) == ns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Test static_group_count().
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_group_count())>);
|
||||
static_assert(noexcept(Mapping::static_group_count()));
|
||||
static_assert(Mapping::static_group_count() == ngroups);
|
||||
|
||||
// Test static_unit_count().
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_unit_count(cuda::std::size_t{}))>);
|
||||
static_assert(noexcept(Mapping::static_unit_count(cuda::std::size_t{})));
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(Mapping::static_unit_count(i) == cuda::std::dynamic_extent);
|
||||
}
|
||||
}
|
||||
|
||||
// Test is_always_exhaustive().
|
||||
static_assert(cuda::std::is_same_v<bool, decltype(Mapping::is_always_exhaustive())>);
|
||||
static_assert(noexcept(Mapping::is_always_exhaustive()));
|
||||
static_assert(!Mapping::is_always_exhaustive());
|
||||
|
||||
// Test unit_count().
|
||||
{
|
||||
static_assert(
|
||||
cuda::std::is_same_v<unsigned, decltype(cuda::std::declval<const Mapping>().unit_count(cuda::std::size_t{}))>);
|
||||
static_assert(noexcept(cuda::std::declval<const Mapping>().unit_count(cuda::std::size_t{})));
|
||||
|
||||
const Mapping mapping{ns, cudax::non_exhaustive};
|
||||
for (cuda::std::size_t i = 0; i < ngroups; ++i)
|
||||
{
|
||||
CHECK(mapping.unit_count(i) == ns[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping{ns, cudax::non_exhaustive};
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
const auto rank_in_warp = cuda::gpu_thread.rank(parent_group);
|
||||
const auto is_valid_ref = (rank_in_warp < ns_sum);
|
||||
|
||||
static_assert(Result::static_group_count() == ngroups);
|
||||
static_assert(Result::static_unit_count() == cuda::std::dynamic_extent);
|
||||
static_assert(!Result::is_always_exhaustive());
|
||||
static_assert(Result::is_always_contiguous());
|
||||
|
||||
CHECK(result.group_count() == static_cast<unsigned>(ngroups));
|
||||
CHECK(result.is_valid() == is_valid_ref);
|
||||
|
||||
if (is_valid_ref)
|
||||
{
|
||||
unsigned group_rank_ref = ngroups - 1;
|
||||
unsigned rank_ref = rank_in_warp - group_starts[group_rank_ref];
|
||||
for (unsigned i = 1; i < ngroups; ++i)
|
||||
{
|
||||
if (rank_in_warp < group_starts[i])
|
||||
{
|
||||
group_rank_ref = i - 1;
|
||||
rank_ref = rank_in_warp - group_starts[i - 1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
CHECK(result.group_rank() == group_rank_ref);
|
||||
|
||||
CHECK(result.unit_count() == ns[group_rank_ref]);
|
||||
CHECK(result.unit_rank() == rank_ref);
|
||||
|
||||
const auto lane_mask_ref =
|
||||
(ns[group_rank_ref] < 32) ? ((1u << ns[group_rank_ref]) - 1) << group_starts[group_rank_ref] : ~0;
|
||||
CHECK(result.lane_mask() == cuda::device::lane_mask{lane_mask_ref});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_group_as<1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1>(
|
||||
config);
|
||||
test_group_as<2, 4, 8, 16, 2>(config);
|
||||
test_group_as<3, 5, 1, 1, 22>(config);
|
||||
test_group_as<31, 1>(config);
|
||||
test_group_as<32>(config);
|
||||
|
||||
test_group_as_non_exhaustive<1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1>(
|
||||
config);
|
||||
test_group_as_non_exhaustive<2, 4, 8, 16, 2>(config);
|
||||
test_group_as_non_exhaustive<3, 5, 1, 1, 22>(config);
|
||||
test_group_as_non_exhaustive<31, 1>(config);
|
||||
test_group_as_non_exhaustive<32>(config);
|
||||
test_group_as_non_exhaustive<31>(config);
|
||||
test_group_as_non_exhaustive<4, 6, 8>(config);
|
||||
test_group_as_non_exhaustive<2, 2, 3, 1, 14>(config);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("Group-as mapping", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<8, 4>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims(dim3{8, 4}));
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,377 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
#include <cuda/warp>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
template <cuda::std::size_t N, class Config>
|
||||
__device__ void test_group_by(Config config)
|
||||
{
|
||||
// Test static N.
|
||||
{
|
||||
using Mapping = cudax::group_by<N>;
|
||||
static_assert(cuda::std::is_same_v<Mapping, cudax::group_by<N, true>>);
|
||||
|
||||
// Test default constructor.
|
||||
{
|
||||
static_assert(cuda::std::is_trivially_default_constructible_v<Mapping>);
|
||||
static_assert(cuda::std::is_empty_v<Mapping>);
|
||||
|
||||
cudax::group_by<N> mapping;
|
||||
CHECK(mapping.unit_count() == static_cast<unsigned>(N));
|
||||
}
|
||||
|
||||
// Test the mapping is not constructible from unsigned.
|
||||
static_assert(!cuda::std::is_constructible_v<Mapping, unsigned>);
|
||||
|
||||
// Test the mapping is not constructible from non_exhaustive_t.
|
||||
static_assert(!cuda::std::is_constructible_v<Mapping, cudax::non_exhaustive_t>);
|
||||
|
||||
// Test the mapping is not constructible from unsigned and non_exhaustive_t.
|
||||
static_assert(!cuda::std::is_constructible_v<Mapping, unsigned, cudax::non_exhaustive_t>);
|
||||
|
||||
// Test static_unit_count().
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_unit_count())>);
|
||||
static_assert(noexcept(Mapping::static_unit_count()));
|
||||
static_assert(Mapping::static_unit_count() == N);
|
||||
|
||||
// Test is_always_exhaustive().
|
||||
static_assert(cuda::std::is_same_v<bool, decltype(Mapping::is_always_exhaustive())>);
|
||||
static_assert(noexcept(Mapping::is_always_exhaustive()));
|
||||
static_assert(Mapping::is_always_exhaustive());
|
||||
|
||||
// Test unit_count().
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<unsigned, decltype(cuda::std::declval<const Mapping>().unit_count())>);
|
||||
static_assert(noexcept(cuda::std::declval<const Mapping>().unit_count()));
|
||||
|
||||
const Mapping mapping;
|
||||
CHECK(mapping.unit_count() == static_cast<unsigned>(N));
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping;
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
static_assert(Result::static_group_count() == 32 / N);
|
||||
CHECK(result.group_count() == cuda::gpu_thread.count(cuda::warp) / N);
|
||||
CHECK(result.group_rank() == cuda::gpu_thread.rank(cuda::warp) / N);
|
||||
|
||||
static_assert(Result::static_unit_count() == N);
|
||||
CHECK(result.unit_count() == N);
|
||||
CHECK(result.unit_rank() == cuda::gpu_thread.rank(cuda::warp) % N);
|
||||
|
||||
const auto lane_mask_ref = ((N < 32) ? ((1u << N) - 1) : ~0u) << ((cuda::gpu_thread.rank(cuda::warp) / N) * N);
|
||||
CHECK(result.lane_mask() == cuda::device::lane_mask{lane_mask_ref});
|
||||
|
||||
CHECK(result.is_valid());
|
||||
static_assert(Result::is_always_exhaustive());
|
||||
static_assert(Result::is_always_contiguous());
|
||||
}
|
||||
}
|
||||
|
||||
// Test dynamic N.
|
||||
{
|
||||
using Mapping = cudax::group_by<>;
|
||||
static_assert(cuda::std::is_same_v<Mapping, cudax::group_by<cuda::std::dynamic_extent, true>>);
|
||||
|
||||
// Test default constructor.
|
||||
static_assert(!cuda::std::is_default_constructible_v<Mapping>);
|
||||
static_assert(!cuda::std::is_empty_v<Mapping>);
|
||||
|
||||
// Test the mapping is constructible from unsigned.
|
||||
{
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<Mapping, unsigned>);
|
||||
|
||||
cudax::group_by mapping{N};
|
||||
static_assert(cuda::std::is_same_v<Mapping, decltype(mapping)>);
|
||||
CHECK(mapping.unit_count() == static_cast<unsigned>(N));
|
||||
}
|
||||
|
||||
// Test the mapping is not constructible from non_exhaustive_t.
|
||||
static_assert(!cuda::std::is_constructible_v<Mapping, cudax::non_exhaustive_t>);
|
||||
|
||||
// Test the mapping is not constructible from unsigned and non_exhaustive_t.
|
||||
static_assert(!cuda::std::is_constructible_v<Mapping, unsigned, cudax::non_exhaustive_t>);
|
||||
|
||||
// Test static_unit_count().
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_unit_count())>);
|
||||
static_assert(noexcept(Mapping::static_unit_count()));
|
||||
static_assert(Mapping::static_unit_count() == cuda::std::dynamic_extent);
|
||||
|
||||
// Test is_always_exhaustive().
|
||||
static_assert(cuda::std::is_same_v<bool, decltype(Mapping::is_always_exhaustive())>);
|
||||
static_assert(noexcept(Mapping::is_always_exhaustive()));
|
||||
static_assert(Mapping::is_always_exhaustive());
|
||||
|
||||
// Test unit_count().
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<unsigned, decltype(cuda::std::declval<const Mapping>().unit_count())>);
|
||||
static_assert(noexcept(cuda::std::declval<const Mapping>().unit_count()));
|
||||
|
||||
const Mapping mapping{N};
|
||||
CHECK(mapping.unit_count() == static_cast<unsigned>(N));
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping{N};
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
static_assert(Result::static_group_count() == cuda::std::dynamic_extent);
|
||||
CHECK(result.group_count() == cuda::gpu_thread.count(cuda::warp) / N);
|
||||
CHECK(result.group_rank() == cuda::gpu_thread.rank(cuda::warp) / N);
|
||||
|
||||
static_assert(Result::static_unit_count() == cuda::std::dynamic_extent);
|
||||
CHECK(result.unit_count() == N);
|
||||
CHECK(result.unit_rank() == cuda::gpu_thread.rank(cuda::warp) % N);
|
||||
|
||||
const auto lane_mask_ref = ((N < 32) ? ((1u << N) - 1) : ~0u) << ((cuda::gpu_thread.rank(cuda::warp) / N) * N);
|
||||
CHECK(result.lane_mask() == cuda::device::lane_mask{lane_mask_ref});
|
||||
|
||||
CHECK(result.is_valid());
|
||||
static_assert(Result::is_always_exhaustive());
|
||||
static_assert(Result::is_always_contiguous());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <cuda::std::size_t N, class Config>
|
||||
__device__ void test_group_by_non_exhaustive(Config config)
|
||||
{
|
||||
// Test static N.
|
||||
{
|
||||
using Mapping = cudax::group_by<N, false>;
|
||||
|
||||
// Test default constructor.
|
||||
static_assert(cuda::std::is_trivially_default_constructible_v<Mapping>);
|
||||
static_assert(cuda::std::is_empty_v<Mapping>);
|
||||
|
||||
// Test the mapping is not constructible from unsigned.
|
||||
static_assert(!cuda::std::is_constructible_v<Mapping, unsigned>);
|
||||
|
||||
// Test the mapping is not constructible from non_exhaustive_t.
|
||||
{
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<Mapping, cudax::non_exhaustive_t>);
|
||||
|
||||
Mapping mapping{cudax::non_exhaustive};
|
||||
static_assert(cuda::std::is_same_v<decltype(mapping), Mapping>);
|
||||
CHECK(mapping.unit_count() == static_cast<unsigned>(N));
|
||||
}
|
||||
|
||||
// Test the mapping is not constructible from unsigned and non_exhaustive_t.
|
||||
static_assert(!cuda::std::is_constructible_v<Mapping, unsigned, cudax::non_exhaustive_t>);
|
||||
|
||||
// Test static_unit_count().
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_unit_count())>);
|
||||
static_assert(noexcept(Mapping::static_unit_count()));
|
||||
static_assert(Mapping::static_unit_count() == N);
|
||||
|
||||
// Test is_always_exhaustive().
|
||||
static_assert(cuda::std::is_same_v<bool, decltype(Mapping::is_always_exhaustive())>);
|
||||
static_assert(noexcept(Mapping::is_always_exhaustive()));
|
||||
static_assert(!Mapping::is_always_exhaustive());
|
||||
|
||||
// Test unit_count().
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<unsigned, decltype(cuda::std::declval<const Mapping>().unit_count())>);
|
||||
static_assert(noexcept(cuda::std::declval<const Mapping>().unit_count()));
|
||||
|
||||
const Mapping mapping{cudax::non_exhaustive};
|
||||
CHECK(mapping.unit_count() == static_cast<unsigned>(N));
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping{cudax::non_exhaustive};
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
static_assert(Result::static_group_count() == 32 / N);
|
||||
static_assert(Result::static_unit_count() == N);
|
||||
static_assert(!Result::is_always_exhaustive());
|
||||
static_assert(Result::is_always_contiguous());
|
||||
|
||||
const auto is_valid_ref = cuda::gpu_thread.rank(cuda::warp) < (cuda::gpu_thread.count(cuda::warp) / N) * N;
|
||||
CHECK(result.is_valid() == is_valid_ref);
|
||||
|
||||
if (is_valid_ref)
|
||||
{
|
||||
CHECK(result.group_count() == cuda::gpu_thread.count(cuda::warp) / N);
|
||||
CHECK(result.group_rank() == cuda::gpu_thread.rank(cuda::warp) / N);
|
||||
|
||||
CHECK(result.unit_count() == N);
|
||||
CHECK(result.unit_rank() == cuda::gpu_thread.rank(cuda::warp) % N);
|
||||
|
||||
const auto lane_mask_ref = ((N < 32) ? ((1u << N) - 1) : ~0u) << ((cuda::gpu_thread.rank(cuda::warp) / N) * N);
|
||||
CHECK(result.lane_mask() == cuda::device::lane_mask{lane_mask_ref});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test dynamic N.
|
||||
{
|
||||
using Mapping = cudax::group_by<cuda::std::dynamic_extent, false>;
|
||||
|
||||
// Test default constructor.
|
||||
static_assert(!cuda::std::is_default_constructible_v<Mapping>);
|
||||
static_assert(!cuda::std::is_empty_v<Mapping>);
|
||||
|
||||
// Test the mapping is constructible from unsigned.
|
||||
{
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<Mapping, unsigned>);
|
||||
|
||||
Mapping mapping{N};
|
||||
static_assert(cuda::std::is_same_v<Mapping, decltype(mapping)>);
|
||||
CHECK(mapping.unit_count() == static_cast<unsigned>(N));
|
||||
}
|
||||
|
||||
// Test the mapping is not constructible from non_exhaustive_t.
|
||||
static_assert(!cuda::std::is_constructible_v<Mapping, cudax::non_exhaustive_t>);
|
||||
|
||||
// Test the mapping is not constructible from unsigned and non_exhaustive_t.
|
||||
{
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<Mapping, unsigned, cudax::non_exhaustive_t>);
|
||||
|
||||
cudax::group_by mapping{static_cast<unsigned>(N), cudax::non_exhaustive};
|
||||
static_assert(cuda::std::is_same_v<decltype(mapping), Mapping>);
|
||||
CHECK(mapping.unit_count() == static_cast<unsigned>(N));
|
||||
}
|
||||
|
||||
// Test static_unit_count().
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_unit_count())>);
|
||||
static_assert(noexcept(Mapping::static_unit_count()));
|
||||
static_assert(Mapping::static_unit_count() == cuda::std::dynamic_extent);
|
||||
|
||||
// Test unit_count().
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<unsigned, decltype(cuda::std::declval<const Mapping>().unit_count())>);
|
||||
static_assert(noexcept(cuda::std::declval<const Mapping>().unit_count()));
|
||||
|
||||
const Mapping mapping{N, cudax::non_exhaustive};
|
||||
CHECK(mapping.unit_count() == static_cast<unsigned>(N));
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping{N, cudax::non_exhaustive};
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
static_assert(Result::static_group_count() == cuda::std::dynamic_extent);
|
||||
static_assert(Result::static_unit_count() == cuda::std::dynamic_extent);
|
||||
static_assert(!Result::is_always_exhaustive());
|
||||
static_assert(Result::is_always_contiguous());
|
||||
|
||||
const auto is_valid_ref = cuda::gpu_thread.rank(cuda::warp) < (cuda::gpu_thread.count(cuda::warp) / N) * N;
|
||||
CHECK(result.is_valid() == is_valid_ref);
|
||||
|
||||
if (is_valid_ref)
|
||||
{
|
||||
CHECK(result.group_count() == cuda::gpu_thread.count(cuda::warp) / N);
|
||||
CHECK(result.group_rank() == cuda::gpu_thread.rank(cuda::warp) / N);
|
||||
|
||||
CHECK(result.unit_count() == N);
|
||||
CHECK(result.unit_rank() == cuda::gpu_thread.rank(cuda::warp) % N);
|
||||
|
||||
const auto lane_mask_ref = ((N < 32) ? ((1u << N) - 1) : ~0u) << ((cuda::gpu_thread.rank(cuda::warp) / N) * N);
|
||||
CHECK(result.lane_mask() == cuda::device::lane_mask{lane_mask_ref});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_group_by<1>(config);
|
||||
test_group_by<2>(config);
|
||||
test_group_by<4>(config);
|
||||
test_group_by<16>(config);
|
||||
test_group_by<32>(config);
|
||||
|
||||
test_group_by_non_exhaustive<1>(config);
|
||||
test_group_by_non_exhaustive<2>(config);
|
||||
test_group_by_non_exhaustive<3>(config);
|
||||
test_group_by_non_exhaustive<4>(config);
|
||||
test_group_by_non_exhaustive<14>(config);
|
||||
test_group_by_non_exhaustive<16>(config);
|
||||
test_group_by_non_exhaustive<30>(config);
|
||||
test_group_by_non_exhaustive<32>(config);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("Group-by mapping", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<8, 4>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims(dim3{8, 4}));
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
#include <cuda/warp>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void test_identity_mapping(Config config)
|
||||
{
|
||||
using Mapping = cudax::identity_mapping;
|
||||
|
||||
// Test default constructor.
|
||||
{
|
||||
static_assert(cuda::std::is_trivially_default_constructible_v<Mapping>);
|
||||
static_assert(cuda::std::is_empty_v<Mapping>);
|
||||
|
||||
[[maybe_unused]] cudax::identity_mapping mapping;
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping;
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
static_assert(cuda::std::is_same_v<Result, ThreadsInWarpMappingResult>);
|
||||
|
||||
static_assert(Result::static_group_count() == ThreadsInWarpMappingResult::static_group_count());
|
||||
CHECK(result.group_count() == prev_mapping_result.group_count());
|
||||
CHECK(result.group_rank() == prev_mapping_result.group_rank());
|
||||
|
||||
static_assert(Result::static_unit_count() == ThreadsInWarpMappingResult::static_unit_count());
|
||||
CHECK(result.unit_count() == prev_mapping_result.unit_count());
|
||||
CHECK(result.unit_rank() == prev_mapping_result.unit_rank());
|
||||
|
||||
CHECK(result.lane_mask() == prev_mapping_result.lane_mask());
|
||||
|
||||
CHECK(result.is_valid() == prev_mapping_result.is_valid());
|
||||
static_assert(Result::is_always_exhaustive() == ThreadsInWarpMappingResult::is_always_exhaustive());
|
||||
static_assert(Result::is_always_contiguous() == ThreadsInWarpMappingResult::is_always_contiguous());
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_identity_mapping(config);
|
||||
test_identity_mapping(config);
|
||||
test_identity_mapping(config);
|
||||
test_identity_mapping(config);
|
||||
test_identity_mapping(config);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("Identity mapping", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<8, 4>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims(dim3{8, 4}));
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,212 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/numeric>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
#include <cuda/warp>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
template <cuda::std::size_t N, class Config>
|
||||
__device__ void test_take(Config config)
|
||||
{
|
||||
constexpr auto n = static_cast<unsigned>(N);
|
||||
|
||||
// Test static N.
|
||||
{
|
||||
using Mapping = cudax::take<N>;
|
||||
|
||||
// Test default constructor.
|
||||
{
|
||||
static_assert(cuda::std::is_trivially_default_constructible_v<Mapping>);
|
||||
static_assert(cuda::std::is_empty_v<Mapping>);
|
||||
|
||||
Mapping mapping;
|
||||
CHECK(mapping.unit_count() == n);
|
||||
}
|
||||
|
||||
// Test static_unit_count().
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_unit_count())>);
|
||||
static_assert(noexcept(Mapping::static_unit_count()));
|
||||
|
||||
static_assert(Mapping::static_unit_count() == N);
|
||||
}
|
||||
|
||||
// Test unit_count().
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<unsigned, decltype(cuda::std::declval<const Mapping>().unit_count())>);
|
||||
static_assert(noexcept(cuda::std::declval<const Mapping>().unit_count()));
|
||||
|
||||
const Mapping mapping;
|
||||
CHECK(mapping.unit_count() == n);
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping;
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
static_assert(Result::static_group_count() == ThreadsInWarpMappingResult::static_group_count());
|
||||
static_assert(Result::static_unit_count() == N);
|
||||
static_assert(Result::is_always_exhaustive()
|
||||
== (Result::static_unit_count() == ThreadsInWarpMappingResult::static_unit_count()));
|
||||
static_assert(Result::is_always_contiguous());
|
||||
|
||||
const auto is_valid_ref = cuda::std::cmp_less(cuda::gpu_thread.rank(cuda::warp), n);
|
||||
CHECK(result.is_valid() == is_valid_ref);
|
||||
|
||||
if (is_valid_ref)
|
||||
{
|
||||
CHECK(result.group_count() == prev_mapping_result.group_count());
|
||||
CHECK(result.group_rank() == prev_mapping_result.group_rank());
|
||||
|
||||
CHECK(result.unit_count() == n);
|
||||
CHECK(result.unit_rank() == cuda::gpu_thread.rank(cuda::warp));
|
||||
|
||||
const auto lane_mask_ref = ((N < 32) ? ((1u << N) - 1) : ~0u);
|
||||
CHECK(result.lane_mask() == cuda::device::lane_mask{lane_mask_ref});
|
||||
|
||||
CHECK(result.is_valid());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test dynamic Ns.
|
||||
{
|
||||
using Mapping = cudax::take<cuda::std::dynamic_extent>;
|
||||
|
||||
// Test default constructor.
|
||||
{
|
||||
static_assert(cuda::std::is_nothrow_default_constructible_v<Mapping>);
|
||||
|
||||
Mapping mapping;
|
||||
CHECK(mapping.unit_count() == 0);
|
||||
}
|
||||
|
||||
// Test the mapping is constructible from n.
|
||||
{
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<Mapping, unsigned>);
|
||||
|
||||
cudax::take mapping{n};
|
||||
static_assert(cuda::std::is_same_v<decltype(mapping), Mapping>);
|
||||
CHECK(mapping.unit_count() == n);
|
||||
}
|
||||
|
||||
// Test static_unit_count().
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<cuda::std::size_t, decltype(Mapping::static_unit_count())>);
|
||||
static_assert(noexcept(Mapping::static_unit_count()));
|
||||
|
||||
static_assert(Mapping::static_unit_count() == cuda::std::dynamic_extent);
|
||||
}
|
||||
|
||||
// Test unit_count().
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<unsigned, decltype(cuda::std::declval<const Mapping>().unit_count())>);
|
||||
static_assert(noexcept(cuda::std::declval<const Mapping>().unit_count()));
|
||||
|
||||
const Mapping mapping{n};
|
||||
CHECK(mapping.unit_count() == n);
|
||||
}
|
||||
|
||||
// Test map(...).
|
||||
{
|
||||
const cudax::this_warp parent_group{config};
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
static_assert(cudax::__group_mapping_result<decltype(cuda::std::declval<const Mapping>().map(
|
||||
cuda::gpu_thread, parent_group, prev_mapping_result))>);
|
||||
static_assert(
|
||||
noexcept(cuda::std::declval<const Mapping>().map(cuda::gpu_thread, parent_group, prev_mapping_result)));
|
||||
|
||||
const Mapping mapping{n};
|
||||
auto result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
using Result = decltype(result);
|
||||
|
||||
static_assert(Result::static_group_count() == ThreadsInWarpMappingResult::static_group_count());
|
||||
static_assert(Result::static_unit_count() == cuda::std::dynamic_extent);
|
||||
static_assert(!Result::is_always_exhaustive());
|
||||
static_assert(Result::is_always_contiguous());
|
||||
|
||||
const auto is_valid_ref = cuda::std::cmp_less(cuda::gpu_thread.rank(cuda::warp), n);
|
||||
CHECK(result.is_valid() == is_valid_ref);
|
||||
|
||||
if (is_valid_ref)
|
||||
{
|
||||
CHECK(result.group_count() == prev_mapping_result.group_count());
|
||||
CHECK(result.group_rank() == prev_mapping_result.group_rank());
|
||||
|
||||
CHECK(result.unit_count() == n);
|
||||
CHECK(result.unit_rank() == cuda::gpu_thread.rank(cuda::warp));
|
||||
|
||||
const auto lane_mask_ref = ((N < 32) ? ((1u << N) - 1) : ~0u);
|
||||
CHECK(result.lane_mask() == cuda::device::lane_mask{lane_mask_ref});
|
||||
|
||||
CHECK(result.is_valid());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_take<0>(config);
|
||||
test_take<1>(config);
|
||||
test_take<2>(config);
|
||||
test_take<3>(config);
|
||||
test_take<4>(config);
|
||||
test_take<14>(config);
|
||||
test_take<16>(config);
|
||||
test_take<30>(config);
|
||||
test_take<32>(config);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("Take mapping", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<8, 4>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims(dim3{8, 4}));
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cub/block/block_reduce.cuh>
|
||||
|
||||
#include <cuda/buffer>
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/iterator>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/algorithm>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/execution>
|
||||
#include <cuda/std/optional>
|
||||
#include <cuda/std/span>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
struct DeviceSegmentedSumKernel
|
||||
{
|
||||
template <class Config, class T, cuda::std::size_t SegmentSize, class GroupFn, class UnitFn>
|
||||
__device__ void operator()(
|
||||
Config config,
|
||||
const T* in,
|
||||
T* out,
|
||||
cuda::std::size_t nsegments,
|
||||
cuda::std::integral_constant<cuda::std::size_t, SegmentSize>,
|
||||
GroupFn group_fn,
|
||||
UnitFn unit_fn)
|
||||
{
|
||||
constexpr auto nitems_per_thread = SegmentSize / cuda::gpu_thread.static_count(cuda::block, config);
|
||||
|
||||
const auto segment_offset = SegmentSize * cuda::block.rank(cuda::grid, config);
|
||||
|
||||
T items[nitems_per_thread];
|
||||
for (cuda::std::size_t i = 0; i < nitems_per_thread; ++i)
|
||||
{
|
||||
const auto offset = cuda::gpu_thread.rank(cuda::block, config) + i * cuda::gpu_thread.count(cuda::block, config);
|
||||
items[i] = *(in + segment_offset + offset);
|
||||
}
|
||||
|
||||
group_fn(cudax::this_block{config}, cuda::std::span{items});
|
||||
|
||||
using BlockReduce = cub::BlockReduce<T, static_cast<int>(cuda::gpu_thread.static_count(cuda::block, config))>;
|
||||
__shared__ typename BlockReduce::TempStorage scratch;
|
||||
|
||||
const auto result = BlockReduce{scratch}.Sum(items);
|
||||
if (cuda::gpu_thread.rank(cuda::block, config) == 0)
|
||||
{
|
||||
out[cuda::block.rank(cuda::grid, config)] = unit_fn(result);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class T, cuda::std::size_t SegmentSize, class GroupFn, class UnitFn>
|
||||
void device_segmented_sum(
|
||||
cuda::stream_ref stream,
|
||||
const T* in,
|
||||
T* out,
|
||||
cuda::std::size_t nsegments,
|
||||
cuda::std::integral_constant<cuda::std::size_t, SegmentSize> segment_size,
|
||||
GroupFn group_fn,
|
||||
UnitFn unit_fn)
|
||||
{
|
||||
const auto config =
|
||||
cuda::make_config(cuda::grid_dims(dim3{static_cast<unsigned>(nsegments)}), cuda::block_dims<SegmentSize>());
|
||||
cuda::launch(stream, config, DeviceSegmentedSumKernel{}, in, out, nsegments, segment_size, group_fn, unit_fn);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("Segmented algorithm", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
auto in = cuda::make_device_buffer<int>(stream, device, 1024, 1);
|
||||
auto out = cuda::make_device_buffer<int>(stream, device, 8, cuda::no_init);
|
||||
|
||||
device_segmented_sum(
|
||||
stream,
|
||||
in.data(),
|
||||
out.data(),
|
||||
8,
|
||||
cuda::std::integral_constant<cuda::std::size_t, 128>{},
|
||||
[] __device__(auto group, auto items) {
|
||||
for (auto& item : items)
|
||||
{
|
||||
item *= 2;
|
||||
}
|
||||
group.sync();
|
||||
},
|
||||
[] __device__(auto value) {
|
||||
return value / 2;
|
||||
});
|
||||
stream.sync();
|
||||
|
||||
CHECK(cuda::std::equal(cuda::execution::gpu, out.begin(), out.end(), cuda::constant_iterator{128}));
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/barrier>
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
template <class Level, class Config>
|
||||
__device__ void test_barrier_synchronizer(const Level& level, Config config)
|
||||
{
|
||||
constexpr cuda::std::size_t nbarriers = 8;
|
||||
|
||||
// Test constructor from static span of barriers.
|
||||
{
|
||||
auto& barriers = get_barriers<nbarriers, 0>(level);
|
||||
using Barrier = cuda::std::remove_all_extents_t<cuda::std::remove_reference_t<decltype(barriers)>>;
|
||||
|
||||
cuda::std::span<Barrier, nbarriers> barriers_span{barriers, nbarriers};
|
||||
cudax::barrier_synchronizer synchronizer{barriers_span};
|
||||
|
||||
static_assert(cuda::std::is_same_v<cudax::barrier_synchronizer<Barrier, nbarriers>, decltype(synchronizer)>);
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<decltype(synchronizer), decltype(barriers_span)>);
|
||||
|
||||
CHECK(synchronizer.barriers().data() == barriers);
|
||||
CHECK(synchronizer.barriers().size() == nbarriers);
|
||||
}
|
||||
|
||||
// Test constructor from dynamic span of barriers.
|
||||
{
|
||||
auto& barriers = get_barriers<nbarriers, 1>(level);
|
||||
using Barrier = cuda::std::remove_all_extents_t<cuda::std::remove_reference_t<decltype(barriers)>>;
|
||||
|
||||
cuda::std::span<Barrier> barriers_span{barriers, nbarriers};
|
||||
cudax::barrier_synchronizer synchronizer{barriers_span};
|
||||
|
||||
static_assert(
|
||||
cuda::std::is_same_v<cudax::barrier_synchronizer<Barrier, cuda::std::dynamic_extent>, decltype(synchronizer)>);
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<decltype(synchronizer), decltype(barriers_span)>);
|
||||
|
||||
CHECK(synchronizer.barriers().data() == barriers);
|
||||
CHECK(synchronizer.barriers().size() == nbarriers);
|
||||
}
|
||||
|
||||
// Test constructor from array of barriers.
|
||||
{
|
||||
auto& barriers = get_barriers<nbarriers, 2>(level);
|
||||
using Barrier = cuda::std::remove_all_extents_t<cuda::std::remove_reference_t<decltype(barriers)>>;
|
||||
|
||||
cudax::barrier_synchronizer synchronizer{barriers};
|
||||
|
||||
static_assert(cuda::std::is_same_v<cudax::barrier_synchronizer<Barrier, nbarriers>, decltype(synchronizer)>);
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<decltype(synchronizer), decltype(barriers)>);
|
||||
|
||||
CHECK(synchronizer.barriers().data() == barriers);
|
||||
CHECK(synchronizer.barriers().size() == nbarriers);
|
||||
}
|
||||
|
||||
// Test barriers().
|
||||
{
|
||||
auto& barriers = get_barriers<nbarriers, 3>(level);
|
||||
using Barrier = cuda::std::remove_all_extents_t<cuda::std::remove_reference_t<decltype(barriers)>>;
|
||||
|
||||
const cudax::barrier_synchronizer synchronizer{barriers};
|
||||
|
||||
static_assert(cuda::std::is_same_v<cuda::std::span<Barrier, nbarriers>, decltype(synchronizer.barriers())>);
|
||||
static_assert(noexcept(synchronizer.barriers()));
|
||||
|
||||
CHECK(synchronizer.barriers().data() == barriers);
|
||||
CHECK(synchronizer.barriers().size() == nbarriers);
|
||||
}
|
||||
|
||||
// Test make_instance(...).
|
||||
{
|
||||
auto& barriers = get_barriers<nbarriers, 4>(level);
|
||||
using Barrier = cuda::std::remove_all_extents_t<cuda::std::remove_reference_t<decltype(barriers)>>;
|
||||
|
||||
const auto parent_group = cudax::make_this_group(level, config);
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
const cudax::group_by mapping{4};
|
||||
const cudax::barrier_synchronizer synchronizer{barriers};
|
||||
|
||||
const auto mapping_result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
const auto synchronizer_instance =
|
||||
synchronizer.make_instance(cuda::gpu_thread, parent_group, mapping, mapping_result);
|
||||
|
||||
// Test do_sync(...).
|
||||
static_assert(cuda::std::is_same_v<void, decltype(synchronizer_instance.do_sync(mapping_result, synchronizer))>);
|
||||
static_assert(noexcept(synchronizer_instance.do_sync(mapping_result, synchronizer)));
|
||||
synchronizer_instance.do_sync(mapping_result, synchronizer);
|
||||
|
||||
// Test do_sync_aligned(...).
|
||||
static_assert(
|
||||
cuda::std::is_same_v<void, decltype(synchronizer_instance.do_sync_aligned(mapping_result, synchronizer))>);
|
||||
static_assert(noexcept(synchronizer_instance.do_sync_aligned(mapping_result, synchronizer)));
|
||||
synchronizer_instance.do_sync_aligned(mapping_result, synchronizer);
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_barrier_synchronizer(cuda::warp, config);
|
||||
test_barrier_synchronizer(cuda::block, config);
|
||||
test_barrier_synchronizer(cuda::cluster, config);
|
||||
test_barrier_synchronizer(cuda::grid, config);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("Barrier synchronizer", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<8, 4>(), cuda::cooperative_launch{});
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
{
|
||||
const auto config =
|
||||
cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims(dim3{8, 4}), cuda::cooperative_launch{});
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/bit>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
template <class Level, class Config>
|
||||
__device__ void test_lane_synchronizer(const Level& level, Config config)
|
||||
{
|
||||
using Synchronizer = cudax::lane_synchronizer;
|
||||
static_assert(cuda::std::is_empty_v<Synchronizer>);
|
||||
|
||||
// Test default constructor.
|
||||
static_assert(cuda::std::is_trivially_default_constructible_v<Synchronizer>);
|
||||
|
||||
// Test make_instance(...).
|
||||
{
|
||||
const auto parent_group = cudax::make_this_group(level, config);
|
||||
const ThreadsInWarpMappingResult prev_mapping_result;
|
||||
|
||||
const cudax::group_by mapping{2};
|
||||
const Synchronizer synchronizer{};
|
||||
|
||||
const auto mapping_result = mapping.map(cuda::gpu_thread, parent_group, prev_mapping_result);
|
||||
const auto synchronizer_instance =
|
||||
synchronizer.make_instance(cuda::gpu_thread, parent_group, mapping, mapping_result);
|
||||
|
||||
// Test do_sync(...).
|
||||
static_assert(cuda::std::is_same_v<void, decltype(synchronizer_instance.do_sync(mapping_result, synchronizer))>);
|
||||
static_assert(noexcept(synchronizer_instance.do_sync(mapping_result, synchronizer)));
|
||||
synchronizer_instance.do_sync(mapping_result, synchronizer);
|
||||
|
||||
// Test do_sync_aligned(...).
|
||||
static_assert(
|
||||
cuda::std::is_same_v<void, decltype(synchronizer_instance.do_sync_aligned(mapping_result, synchronizer))>);
|
||||
static_assert(noexcept(synchronizer_instance.do_sync_aligned(mapping_result, synchronizer)));
|
||||
synchronizer_instance.do_sync_aligned(mapping_result, synchronizer);
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_lane_synchronizer(cuda::warp, config);
|
||||
test_lane_synchronizer(cuda::block, config);
|
||||
test_lane_synchronizer(cuda::cluster, config);
|
||||
test_lane_synchronizer(cuda::grid, config);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("Lane synchronizer", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<8, 4>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims(dim3{8, 4}));
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
}
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,324 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/atomic>
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include <cooperative_groups.h>
|
||||
|
||||
#include "group_testing.cuh"
|
||||
|
||||
namespace
|
||||
{
|
||||
__device__ unsigned global_var = 0;
|
||||
|
||||
template <class Level, class Hierarchy, class Group>
|
||||
__device__ void test_common_properties(const Hierarchy&, Group& group)
|
||||
{
|
||||
// Assert that Group satisfies the group concept.
|
||||
static_assert(cudax::is_group<Group>);
|
||||
|
||||
// Test types
|
||||
static_assert(cuda::std::is_same_v<Level, typename Group::unit_type>);
|
||||
static_assert(cuda::std::is_same_v<Level, typename Group::level_type>);
|
||||
|
||||
// Test that the group can be queried for it's hierarchy.
|
||||
{
|
||||
decltype(auto) hierarchy = cuda::std::as_const(group).hierarchy();
|
||||
static_assert(cuda::std::is_same_v<decltype(hierarchy), const Hierarchy&>);
|
||||
}
|
||||
|
||||
// Test that the group can be synchronized using .sync() method.
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<void, decltype(group.sync())>);
|
||||
static_assert(noexcept(group.sync()));
|
||||
|
||||
// .sync() method must support calls from different branches. Add some dummy work to make sure the branches are not
|
||||
// collided.
|
||||
cuda::atomic_ref<unsigned, cuda::thread_scope_device> atomic{global_var};
|
||||
if ((threadIdx.x + threadIdx.y + threadIdx.z) % 2 == 0)
|
||||
{
|
||||
atomic++;
|
||||
group.sync();
|
||||
atomic--;
|
||||
}
|
||||
else
|
||||
{
|
||||
atomic--;
|
||||
group.sync();
|
||||
atomic++;
|
||||
}
|
||||
}
|
||||
|
||||
// Test that the group can be synchronized using .sync_aligned() method.
|
||||
{
|
||||
static_assert(cuda::std::is_same_v<void, decltype(group.sync_aligned())>);
|
||||
static_assert(noexcept(group.sync_aligned()));
|
||||
|
||||
// .sync_aligned() method must be called by all threads in the group uniformly in one place.
|
||||
group.sync_aligned();
|
||||
}
|
||||
}
|
||||
|
||||
template <class Hierarchy>
|
||||
__device__ void test_this_queries(const cudax::this_thread<Hierarchy>& group)
|
||||
{
|
||||
// todo(dabayer): These queries end up in `error: expression must have a constant value`, when group is taken by
|
||||
// reference. Can we find a solution that works without copying the group?
|
||||
// static_assert(cuda::gpu_thread.static_count(group) == 1);
|
||||
|
||||
REQUIRE(cuda::gpu_thread.count(group) == 1);
|
||||
REQUIRE(group.count(cuda::warp) == cuda::gpu_thread.count(cuda::warp));
|
||||
REQUIRE(group.count(cuda::block) == cuda::gpu_thread.count(cuda::block));
|
||||
REQUIRE(group.count(cuda::cluster) == cuda::gpu_thread.count(cuda::cluster));
|
||||
REQUIRE(group.count(cuda::grid) == cuda::gpu_thread.count(cuda::grid));
|
||||
|
||||
REQUIRE(cuda::gpu_thread.rank(group) == 0);
|
||||
REQUIRE(group.rank(cuda::warp) == cuda::gpu_thread.rank(cuda::warp));
|
||||
REQUIRE(group.rank(cuda::block) == cuda::gpu_thread.rank(cuda::block));
|
||||
REQUIRE(group.rank(cuda::cluster) == cuda::gpu_thread.rank(cuda::cluster));
|
||||
REQUIRE(group.rank(cuda::grid) == cuda::gpu_thread.rank(cuda::grid));
|
||||
|
||||
REQUIRE(cuda::gpu_thread.is_root_rank(group));
|
||||
|
||||
REQUIRE(cuda::gpu_thread.is_part_of(group));
|
||||
}
|
||||
|
||||
template <class Hierarchy>
|
||||
__device__ void test_this_queries(const cudax::this_warp<Hierarchy>& group)
|
||||
{
|
||||
// todo(dabayer): These queries end up in `error: expression must have a constant value`, when group is taken by
|
||||
// reference. Can we find a solution that works without copying the group?
|
||||
// static_assert(cuda::gpu_thread.static_count(group) == cuda::gpu_thread.static_count(cuda::warp,
|
||||
// group.hierarchy())); static_assert(cuda::warp.static_count(group) == 1);
|
||||
|
||||
REQUIRE(cuda::gpu_thread.count(group) == cuda::gpu_thread.count(cuda::warp));
|
||||
REQUIRE(cuda::warp.count(group) == 1);
|
||||
REQUIRE(group.count(cuda::block) == cuda::warp.count(cuda::block));
|
||||
REQUIRE(group.count(cuda::cluster) == cuda::warp.count(cuda::cluster));
|
||||
REQUIRE(group.count(cuda::grid) == cuda::warp.count(cuda::grid));
|
||||
|
||||
REQUIRE(cuda::gpu_thread.rank(group) == cuda::gpu_thread.rank(cuda::warp));
|
||||
REQUIRE(cuda::warp.rank(group) == 0);
|
||||
REQUIRE(group.rank(cuda::block) == cuda::warp.rank(cuda::block));
|
||||
REQUIRE(group.rank(cuda::cluster) == cuda::warp.rank(cuda::cluster));
|
||||
REQUIRE(group.rank(cuda::grid) == cuda::warp.rank(cuda::grid));
|
||||
|
||||
REQUIRE(cuda::gpu_thread.is_root_rank(group) == (cuda::gpu_thread.rank(cuda::warp) == 0));
|
||||
REQUIRE(cuda::warp.is_root_rank(group));
|
||||
|
||||
REQUIRE(cuda::gpu_thread.is_part_of(group));
|
||||
REQUIRE(cuda::warp.is_part_of(group));
|
||||
}
|
||||
|
||||
template <class Hierarchy>
|
||||
__device__ void test_this_queries(const cudax::this_block<Hierarchy>& group)
|
||||
{
|
||||
// todo(dabayer): These queries end up in `error: expression must have a constant value`, when group is taken by
|
||||
// reference. Can we find a solution that works without copying the group?
|
||||
// static_assert(cuda::gpu_thread.static_count(group) == cuda::gpu_thread.static_count(cuda::block,
|
||||
// group.hierarchy())); static_assert(cuda::warp.static_count(group) == cuda::warp.static_count(cuda::block,
|
||||
// group.hierarchy())); static_assert(cuda::block.static_count(group) == 1);
|
||||
|
||||
REQUIRE(cuda::gpu_thread.count(group) == cuda::gpu_thread.count(cuda::block));
|
||||
REQUIRE(cuda::warp.count(group) == cuda::warp.count(cuda::block));
|
||||
REQUIRE(cuda::block.count(group) == 1);
|
||||
REQUIRE(group.count(cuda::cluster) == cuda::block.count(cuda::cluster));
|
||||
REQUIRE(group.count(cuda::grid) == cuda::block.count(cuda::grid));
|
||||
|
||||
REQUIRE(cuda::gpu_thread.rank(group) == cuda::gpu_thread.rank(cuda::block));
|
||||
REQUIRE(cuda::warp.rank(group) == cuda::warp.rank(cuda::block));
|
||||
REQUIRE(cuda::block.rank(group) == 0);
|
||||
REQUIRE(group.rank(cuda::cluster) == cuda::block.rank(cuda::cluster));
|
||||
REQUIRE(group.rank(cuda::grid) == cuda::block.rank(cuda::grid));
|
||||
|
||||
REQUIRE(cuda::gpu_thread.is_root_rank(group) == (cuda::gpu_thread.rank(cuda::block) == 0));
|
||||
REQUIRE(cuda::warp.is_root_rank(group) == (cuda::warp.rank(cuda::block) == 0));
|
||||
REQUIRE(cuda::block.is_root_rank(group));
|
||||
|
||||
REQUIRE(cuda::gpu_thread.is_part_of(group));
|
||||
REQUIRE(cuda::warp.is_part_of(group));
|
||||
REQUIRE(cuda::block.is_part_of(group));
|
||||
}
|
||||
|
||||
template <class Hierarchy>
|
||||
__device__ void test_this_queries(const cudax::this_cluster<Hierarchy>& group)
|
||||
{
|
||||
// todo(dabayer): These queries end up in `error: expression must have a constant value`, when group is taken by
|
||||
// reference. Can we find a solution that works without copying the group?
|
||||
// static_assert(cuda::gpu_thread.static_count(group) == cuda::gpu_thread.static_count(cuda::cluster,
|
||||
// group.hierarchy())); static_assert(cuda::warp.static_count(group) == cuda::warp.static_count(cuda::cluster,
|
||||
// group.hierarchy())); static_assert(cuda::block.static_count(group) == cuda::block.static_count(cuda::cluster,
|
||||
// group.hierarchy())); static_assert(cuda::cluster.static_count(group) == 1);
|
||||
|
||||
REQUIRE(cuda::gpu_thread.count(group) == cuda::gpu_thread.count(cuda::cluster));
|
||||
REQUIRE(cuda::warp.count(group) == cuda::warp.count(cuda::cluster));
|
||||
REQUIRE(cuda::block.count(group) == cuda::block.count(cuda::cluster));
|
||||
REQUIRE(cuda::cluster.count(group) == 1);
|
||||
REQUIRE(group.count(cuda::grid) == cuda::cluster.count(cuda::grid));
|
||||
|
||||
REQUIRE(cuda::gpu_thread.rank(group) == cuda::gpu_thread.rank(cuda::cluster));
|
||||
REQUIRE(cuda::warp.rank(group) == cuda::warp.rank(cuda::cluster));
|
||||
REQUIRE(cuda::block.rank(group) == cuda::block.rank(cuda::cluster));
|
||||
REQUIRE(cuda::cluster.rank(group) == 0);
|
||||
REQUIRE(group.rank(cuda::grid) == cuda::cluster.rank(cuda::grid));
|
||||
|
||||
REQUIRE(cuda::gpu_thread.is_root_rank(group) == (cuda::gpu_thread.rank(cuda::cluster) == 0));
|
||||
REQUIRE(cuda::warp.is_root_rank(group) == (cuda::warp.rank(cuda::cluster) == 0));
|
||||
REQUIRE(cuda::block.is_root_rank(group) == (cuda::block.rank(cuda::cluster) == 0));
|
||||
REQUIRE(cuda::cluster.is_root_rank(group));
|
||||
|
||||
REQUIRE(cuda::gpu_thread.is_part_of(group));
|
||||
REQUIRE(cuda::warp.is_part_of(group));
|
||||
REQUIRE(cuda::block.is_part_of(group));
|
||||
REQUIRE(cuda::cluster.is_part_of(group));
|
||||
}
|
||||
|
||||
template <class Hierarchy>
|
||||
__device__ void test_this_queries(const cudax::this_grid<Hierarchy>& group)
|
||||
{
|
||||
// todo(dabayer): These queries end up in `error: expression must have a constant value`, when group is taken by
|
||||
// reference. Can we find a solution that works without copying the group?
|
||||
// static_assert(cuda::gpu_thread.static_count(group) == cuda::gpu_thread.static_count(cuda::grid,
|
||||
// group.hierarchy())); static_assert(cuda::warp.static_count(group) == cuda::warp.static_count(cuda::grid,
|
||||
// group.hierarchy())); static_assert(cuda::block.static_count(group) == cuda::block.static_count(cuda::grid,
|
||||
// group.hierarchy())); static_assert(cuda::cluster.static_count(group) == cuda::cluster.static_count(cuda::grid,
|
||||
// group.hierarchy())); static_assert(cuda::grid.static_count(group) == 1);
|
||||
|
||||
REQUIRE(cuda::gpu_thread.count(group) == cuda::gpu_thread.count(cuda::grid));
|
||||
REQUIRE(cuda::warp.count(group) == cuda::warp.count(cuda::grid));
|
||||
REQUIRE(cuda::block.count(group) == cuda::block.count(cuda::grid));
|
||||
REQUIRE(cuda::cluster.count(group) == cuda::cluster.count(cuda::grid));
|
||||
REQUIRE(cuda::grid.count(group) == 1);
|
||||
|
||||
REQUIRE(cuda::gpu_thread.rank(group) == cuda::gpu_thread.rank(cuda::grid));
|
||||
REQUIRE(cuda::warp.rank(group) == cuda::warp.rank(cuda::grid));
|
||||
REQUIRE(cuda::block.rank(group) == cuda::block.rank(cuda::grid));
|
||||
REQUIRE(cuda::cluster.rank(group) == cuda::cluster.rank(cuda::grid));
|
||||
REQUIRE(cuda::grid.rank(group) == 0);
|
||||
|
||||
REQUIRE(cuda::gpu_thread.is_root_rank(group) == (cuda::gpu_thread.rank(cuda::grid) == 0));
|
||||
REQUIRE(cuda::warp.is_root_rank(group) == (cuda::warp.rank(cuda::grid) == 0));
|
||||
REQUIRE(cuda::block.is_root_rank(group) == (cuda::block.rank(cuda::grid) == 0));
|
||||
REQUIRE(cuda::cluster.is_root_rank(group) == (cuda::cluster.rank(cuda::grid) == 0));
|
||||
REQUIRE(cuda::grid.is_root_rank(group));
|
||||
|
||||
REQUIRE(cuda::gpu_thread.is_part_of(group));
|
||||
REQUIRE(cuda::warp.is_part_of(group));
|
||||
REQUIRE(cuda::block.is_part_of(group));
|
||||
REQUIRE(cuda::cluster.is_part_of(group));
|
||||
REQUIRE(cuda::grid.is_part_of(group));
|
||||
}
|
||||
|
||||
template <class Level, class Hierarchy>
|
||||
__device__ void test_cg_interop(const Hierarchy& hierarchy)
|
||||
{
|
||||
if constexpr (cuda::std::is_same_v<Level, cuda::thread_level>)
|
||||
{
|
||||
cudax::this_thread group{cooperative_groups::this_thread()};
|
||||
test_common_properties<Level>(hierarchy, group);
|
||||
}
|
||||
else if constexpr (cuda::std::is_same_v<Level, cuda::warp_level>)
|
||||
{
|
||||
cudax::this_warp group{cooperative_groups::tiled_partition<32>(cooperative_groups::this_thread_block())};
|
||||
test_common_properties<Level>(hierarchy, group);
|
||||
}
|
||||
else if constexpr (cuda::std::is_same_v<Level, cuda::block_level>)
|
||||
{
|
||||
cudax::this_block group{cooperative_groups::this_thread_block()};
|
||||
test_common_properties<Level>(hierarchy, group);
|
||||
}
|
||||
else if constexpr (cuda::std::is_same_v<Level, cuda::cluster_level>)
|
||||
{
|
||||
#if defined(_CG_HAS_CLUSTER_GROUP)
|
||||
NV_IF_TARGET(NV_PROVIDES_SM_90, ({
|
||||
cudax::this_cluster group{cooperative_groups::this_cluster()};
|
||||
test_common_properties<Level>(hierarchy, group);
|
||||
}))
|
||||
#endif // _CG_HAS_CLUSTER_GROUP
|
||||
}
|
||||
else if constexpr (cuda::std::is_same_v<Level, cuda::grid_level>)
|
||||
{
|
||||
cudax::this_grid group{cooperative_groups::this_grid()};
|
||||
test_common_properties<Level>(hierarchy, group);
|
||||
}
|
||||
}
|
||||
|
||||
template <template <class> class GroupTempl, class Level, class Config>
|
||||
__device__ void test_this_group(const Level& level, const Config& config)
|
||||
{
|
||||
const auto implicit_hierarchy = cudax::__implicit_hierarchy();
|
||||
|
||||
// Test implicit construction.
|
||||
{
|
||||
GroupTempl group;
|
||||
static_assert(cuda::std::is_same_v<GroupTempl<cudax::__implicit_hierarchy_t>, decltype(group)>);
|
||||
static_assert(cuda::std::is_nothrow_default_constructible_v<decltype(group)>);
|
||||
|
||||
test_common_properties<Level>(implicit_hierarchy, group);
|
||||
test_this_queries(group);
|
||||
}
|
||||
|
||||
// Test construction from kernel_config.
|
||||
{
|
||||
GroupTempl group{config};
|
||||
// nvcc 12.0 doesn't evaluate these static asserts correctly
|
||||
#if !_CCCL_CUDA_COMPILER(NVCC, ==, 12, 0)
|
||||
static_assert(cuda::std::is_same_v<GroupTempl<typename Config::hierarchy_type>, decltype(group)>);
|
||||
static_assert(cuda::std::is_nothrow_constructible_v<decltype(group), const typename Config::hierarchy_type&>);
|
||||
#endif // !_CCCL_CUDA_COMPILER(NVCC, ==, 12, 0)
|
||||
|
||||
test_common_properties<Level>(config.hierarchy(), group);
|
||||
test_this_queries(group);
|
||||
}
|
||||
|
||||
// Test construction from CG equivalents.
|
||||
test_cg_interop<Level>(implicit_hierarchy);
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_this_group<cudax::this_thread>(cuda::gpu_thread, config);
|
||||
test_this_group<cudax::this_warp>(cuda::warp, config);
|
||||
test_this_group<cudax::this_block>(cuda::block, config);
|
||||
test_this_group<cudax::this_cluster>(cuda::cluster, config);
|
||||
test_this_group<cudax::this_grid>(cuda::grid, config);
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
C2H_TEST("This group", "[group]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
const auto config = cuda::make_config(cuda::grid_dims<2>(), cuda::block_dims<128>(), cuda::cooperative_launch{});
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
|
||||
if (cuda::device_attributes::compute_capability(device) >= cuda::compute_capability{90})
|
||||
{
|
||||
const auto config_cluster = cuda::make_config(
|
||||
cuda::grid_dims<2>(), cuda::cluster_dims<3>(), cuda::block_dims<128>(), cuda::cooperative_launch{});
|
||||
cuda::launch(stream, config_cluster, TestKernel{});
|
||||
}
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
Reference in New Issue
Block a user