[CCCL] 瘦身 + 补全: 移除 cudax/python/libcudacxx-tests 冗余文件, 新增 c2h 测试助手 + cmake 构建系统 + 8 个 CUDA thrust examples
变更摘要:
- 删除: cudax/ (783 files, 7.2M) — 实验性组件，竞赛不需要
- 删除: python/ (226 files, 2.0M) — Python 绑定，竞赛不需要
- 删除: libcudacxx/{test,benchmarks,codegen,cmake,share} (4432 files, 31M)
保留: libcudacxx/include/ (1463 headers, cuda::std 编译依赖)
- 新增: c2h/ (27 files) — CUB Catch2 测试辅助头文件，编译 243 个测试必需
- 新增: cmake/ (29 files) — CCCL 原生 CMake 构建系统
- 新增: thrust/examples/cuda/ (7 files) + cpp_integration/ (1 file)
async_reduce, custom_temporary_allocation, explicit_cuda_stream,
global_device_vector, range_view, unwrap_pointer, wrap_pointer, device
结果: cccl_upstream 从 74M→35M (瘦身 53%), 核心内容 100% 保留:
27/27 tuning headers, 78 benchmarks, 243 tests,
60 thrust examples, 18 CUB examples, 全部编译头文件
This commit is contained in:
muh-bot
@@ -1,139 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/coop.cuh>
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "testing.cuh"
|
||||
|
||||
template <class Group>
|
||||
__device__ void test_group(const Group& group)
|
||||
{
|
||||
// Single value tests.
|
||||
{
|
||||
// Test all threads with false.
|
||||
{
|
||||
const auto result = cudax::coop::any_of(group, false);
|
||||
REQUIRE(result.has_value());
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(!result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads with true.
|
||||
{
|
||||
const auto result = cudax::coop::any_of(group, true);
|
||||
REQUIRE(result.has_value());
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(result.value());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
// Test all threads with false (broadcasted).
|
||||
{
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, false);
|
||||
REQUIRE(!result);
|
||||
}
|
||||
|
||||
// Test all threads with true (broadcasted).
|
||||
{
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, true);
|
||||
REQUIRE(result);
|
||||
}
|
||||
}
|
||||
|
||||
// Array tests.
|
||||
{
|
||||
// Test all threads with false.
|
||||
{
|
||||
bool in[]{false, false, false};
|
||||
const auto result = cudax::coop::any_of(group, in);
|
||||
REQUIRE(result.has_value());
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(!result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads with true.
|
||||
{
|
||||
bool in[]{true, true, true, true};
|
||||
const auto result = cudax::coop::any_of(group, in);
|
||||
REQUIRE(result.has_value());
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads except 1 with false.
|
||||
{
|
||||
bool in[]{false, false, true};
|
||||
const auto result = cudax::coop::any_of(group, in);
|
||||
REQUIRE(result.has_value());
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads with false (broadcasted).
|
||||
{
|
||||
bool in[]{false, false, false};
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
|
||||
REQUIRE(!result);
|
||||
}
|
||||
|
||||
// Test all threads with true (broadcasted).
|
||||
{
|
||||
bool in[]{true, true, true, true};
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
|
||||
REQUIRE(result);
|
||||
}
|
||||
|
||||
// Test all threads except 1 with false (broadcasted).
|
||||
{
|
||||
bool in[]{false, false, true};
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
|
||||
REQUIRE(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_group(cudax::this_thread{});
|
||||
test_group(cudax::this_thread{config});
|
||||
}
|
||||
};
|
||||
|
||||
C2H_TEST("any_of/this_thread", "[any_of][this_thread]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/coop.cuh>
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "testing.cuh"
|
||||
|
||||
template <class Group>
|
||||
__device__ void test_group(const Group& group)
|
||||
{
|
||||
const auto my_rank = cuda::gpu_thread.rank_as<unsigned>(group);
|
||||
|
||||
// Single value tests.
|
||||
{
|
||||
// Test all threads with false.
|
||||
{
|
||||
const auto result = cudax::coop::any_of(group, false);
|
||||
REQUIRE(result.has_value() == (my_rank == 0));
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(!result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads with true.
|
||||
{
|
||||
const auto result = cudax::coop::any_of(group, true);
|
||||
REQUIRE(result.has_value() == (my_rank == 0));
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads except 1 with false.
|
||||
{
|
||||
const auto result = cudax::coop::any_of(group, (my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1));
|
||||
REQUIRE(result.has_value() == (my_rank == 0));
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads with false (broadcasted).
|
||||
{
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, false);
|
||||
REQUIRE(!result);
|
||||
}
|
||||
|
||||
// Test all threads with true (broadcasted).
|
||||
{
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, true);
|
||||
REQUIRE(result);
|
||||
}
|
||||
|
||||
// Test all threads except 1 with false (broadcasted).
|
||||
{
|
||||
const bool result =
|
||||
cudax::coop::any_of(cudax::broadcasted, group, (my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1));
|
||||
REQUIRE(result);
|
||||
}
|
||||
}
|
||||
|
||||
// Array tests.
|
||||
{
|
||||
// Test all threads with false.
|
||||
{
|
||||
bool in[]{false, false, false};
|
||||
const auto result = cudax::coop::any_of(group, in);
|
||||
REQUIRE(result.has_value() == (my_rank == 0));
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(!result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads with true.
|
||||
{
|
||||
bool in[]{true, true, true, true};
|
||||
const auto result = cudax::coop::any_of(group, in);
|
||||
REQUIRE(result.has_value() == (my_rank == 0));
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads except 1 with false.
|
||||
{
|
||||
bool in[]{false, false, my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1};
|
||||
const auto result = cudax::coop::any_of(group, in);
|
||||
REQUIRE(result.has_value() == (my_rank == 0));
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads with false (broadcasted).
|
||||
{
|
||||
bool in[]{false, false, false};
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
|
||||
REQUIRE(!result);
|
||||
}
|
||||
|
||||
// Test all threads with true (broadcasted).
|
||||
{
|
||||
bool in[]{true, true, true, true};
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
|
||||
REQUIRE(result);
|
||||
}
|
||||
|
||||
// Test all threads except 1 with false (broadcasted).
|
||||
{
|
||||
bool in[]{false, false, my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1};
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
|
||||
REQUIRE(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_group(cudax::this_warp{});
|
||||
test_group(cudax::this_warp{config});
|
||||
}
|
||||
};
|
||||
|
||||
C2H_TEST("any_of/this_warp", "[any_of][this_warp]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,187 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/coop.cuh>
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "testing.cuh"
|
||||
|
||||
template <class Group>
|
||||
__device__ void test_group(const Group& group)
|
||||
{
|
||||
// Exit all threads that are not part of the group.
|
||||
if (!cuda::gpu_thread.is_part_of(group))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const auto my_rank = cuda::gpu_thread.rank_as<unsigned>(group);
|
||||
|
||||
// Single value tests.
|
||||
{
|
||||
// Test all threads with false.
|
||||
{
|
||||
const auto result = cudax::coop::any_of(group, false);
|
||||
REQUIRE(result.has_value() == (my_rank == 0));
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(!result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads with true.
|
||||
{
|
||||
const auto result = cudax::coop::any_of(group, true);
|
||||
REQUIRE(result.has_value() == (my_rank == 0));
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads except 1 with false.
|
||||
{
|
||||
const auto result = cudax::coop::any_of(group, (my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1));
|
||||
REQUIRE(result.has_value() == (my_rank == 0));
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads with false (broadcasted).
|
||||
{
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, false);
|
||||
REQUIRE(!result);
|
||||
}
|
||||
|
||||
// Test all threads with true (broadcasted).
|
||||
{
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, true);
|
||||
REQUIRE(result);
|
||||
}
|
||||
|
||||
// Test all threads except 1 with false (broadcasted).
|
||||
{
|
||||
const bool result =
|
||||
cudax::coop::any_of(cudax::broadcasted, group, (my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1));
|
||||
REQUIRE(result);
|
||||
}
|
||||
}
|
||||
|
||||
// Array tests.
|
||||
{
|
||||
// Test all threads with false.
|
||||
{
|
||||
bool in[]{false, false, false};
|
||||
const auto result = cudax::coop::any_of(group, in);
|
||||
REQUIRE(result.has_value() == (my_rank == 0));
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(!result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads with true.
|
||||
{
|
||||
bool in[]{true, true, true, true};
|
||||
const auto result = cudax::coop::any_of(group, in);
|
||||
REQUIRE(result.has_value() == (my_rank == 0));
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads except 1 with false.
|
||||
{
|
||||
bool in[]{false, false, my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1};
|
||||
const auto result = cudax::coop::any_of(group, in);
|
||||
REQUIRE(result.has_value() == (my_rank == 0));
|
||||
if (result.has_value())
|
||||
{
|
||||
REQUIRE(result.value());
|
||||
}
|
||||
}
|
||||
|
||||
// Test all threads with false (broadcasted).
|
||||
{
|
||||
bool in[]{false, false, false};
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
|
||||
REQUIRE(!result);
|
||||
}
|
||||
|
||||
// Test all threads with true (broadcasted).
|
||||
{
|
||||
bool in[]{true, true, true, true};
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
|
||||
REQUIRE(result);
|
||||
}
|
||||
|
||||
// Test all threads except 1 with false (broadcasted).
|
||||
{
|
||||
bool in[]{false, false, my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1};
|
||||
const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
|
||||
REQUIRE(result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct CustomBinaryPartition
|
||||
{
|
||||
template <class MappingResult>
|
||||
__device__ bool operator()(MappingResult mapping_result)
|
||||
{
|
||||
switch (mapping_result.unit_rank())
|
||||
{
|
||||
case 1:
|
||||
case 5:
|
||||
case 14:
|
||||
case 15:
|
||||
case 31:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
const cudax::this_warp warp{config};
|
||||
test_group(cudax::group{cuda::gpu_thread, warp, cudax::identity_mapping{}, cudax::lane_synchronizer{}});
|
||||
test_group(cudax::group{cuda::gpu_thread, warp, cudax::group_by<4>{}, cudax::lane_synchronizer{}});
|
||||
test_group(cudax::group{cuda::gpu_thread, warp, cudax::group_by{1}, cudax::lane_synchronizer{}});
|
||||
test_group(
|
||||
cudax::group{cuda::gpu_thread, warp, cudax::group_by{3, cudax::non_exhaustive}, cudax::lane_synchronizer{}});
|
||||
test_group(cudax::group{
|
||||
cuda::gpu_thread, warp, cudax::binary_partition{CustomBinaryPartition{}}, cudax::lane_synchronizer{}});
|
||||
}
|
||||
};
|
||||
|
||||
C2H_TEST("any_of/threads_within_warp", "[any_of][threads_within_warp]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,223 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/functional>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/algorithm>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/coop.cuh>
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include <testing.cuh>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <c2h/extended_types.h>
|
||||
#include <c2h/generators.h>
|
||||
#include <catch2/matchers/catch_matchers_floating_point.hpp>
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Thread Reduce Wrapper Kernels
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <bool Broadcasted>
|
||||
struct ReduceKernel
|
||||
{
|
||||
template <class Config, int NumItems, class T, class RedOp>
|
||||
__device__ void operator()(
|
||||
Config config,
|
||||
cuda::std::integral_constant<int, NumItems>,
|
||||
const T* __restrict__ d_in,
|
||||
T* __restrict__ d_out,
|
||||
RedOp red_op)
|
||||
{
|
||||
cudax::this_block block{config};
|
||||
|
||||
T thread_data[NumItems];
|
||||
for (int i = 0; i < NumItems; ++i)
|
||||
{
|
||||
thread_data[i] = d_in[cuda::gpu_thread.rank_as<int>(block) + i * cuda::gpu_thread.count_as<int>(block)];
|
||||
}
|
||||
|
||||
if constexpr (Broadcasted)
|
||||
{
|
||||
//! [broadcasted reduce]
|
||||
// Every thread in the block receives the same reduction result.
|
||||
const auto result = cudax::coop::reduce(cudax::broadcasted, block, thread_data, red_op);
|
||||
//! [broadcasted reduce]
|
||||
|
||||
d_out[cuda::gpu_thread.rank(block)] = result;
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto result = cudax::coop::reduce(block, thread_data, red_op);
|
||||
|
||||
REQUIRE(result.has_value() == cuda::gpu_thread.is_root_rank(block));
|
||||
if (cuda::gpu_thread.is_root_rank(block))
|
||||
{
|
||||
*d_out = result.value();
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Type list definition
|
||||
**********************************************************************************************************************/
|
||||
|
||||
using integral_type_list =
|
||||
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
|
||||
|
||||
using fp_type_list = c2h::type_list<float, double>;
|
||||
|
||||
using operator_integral_list =
|
||||
c2h::type_list<cuda::std::plus<>,
|
||||
cuda::std::multiplies<>,
|
||||
cuda::std::bit_and<>,
|
||||
cuda::std::bit_or<>,
|
||||
cuda::std::bit_xor<>,
|
||||
cuda::minimum<>,
|
||||
cuda::maximum<>>;
|
||||
|
||||
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
|
||||
|
||||
using block_size_list = c2h::enum_type_list<int, 3, 32, 63, 128>;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Verify results and kernel launch
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <class T>
|
||||
void verify_results(const T& expected_data, const T& test_results)
|
||||
{
|
||||
if constexpr (cuda::std::is_floating_point_v<T>)
|
||||
{
|
||||
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
|
||||
}
|
||||
else
|
||||
{
|
||||
REQUIRE(expected_data == test_results);
|
||||
}
|
||||
}
|
||||
|
||||
template <int BlockSize, class T, class RedOp, bool Broadcasted = false>
|
||||
void run_reduce_kernel(
|
||||
cuda::stream_ref stream,
|
||||
cuda::std::integral_constant<int, BlockSize>,
|
||||
int num_items,
|
||||
const c2h::device_vector<T>& in,
|
||||
c2h::device_vector<T>& out,
|
||||
RedOp red_op,
|
||||
cuda::std::bool_constant<Broadcasted> = {})
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<BlockSize>());
|
||||
const auto in_ptr = thrust::raw_pointer_cast(in.data());
|
||||
const auto out_ptr = thrust::raw_pointer_cast(out.data());
|
||||
const ReduceKernel<Broadcasted> kernel{};
|
||||
|
||||
switch (num_items)
|
||||
{
|
||||
case 1:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 1>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 4:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 4>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
default:
|
||||
FAIL("Unsupported number of items");
|
||||
}
|
||||
stream.sync();
|
||||
}
|
||||
|
||||
constexpr int max_size = 4;
|
||||
constexpr int num_seeds = 10;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Test cases
|
||||
**********************************************************************************************************************/
|
||||
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
|
||||
|
||||
C2H_TEST("reduce/this_block Integral Type Tests",
|
||||
"[reduce][this_block]",
|
||||
integral_type_list,
|
||||
operator_integral_list,
|
||||
block_size_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
using block_size_t = c2h::get<2, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * block_size_t::value);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items : {1, 4})
|
||||
{
|
||||
auto reference_result =
|
||||
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * block_size_t::value, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, block_size_t{}, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("reduce/this_block Floating-Point Type Tests",
|
||||
"[reduce][this_block]",
|
||||
fp_type_list,
|
||||
operator_fp_list,
|
||||
block_size_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
using block_size_t = c2h::get<2, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
const auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * block_size_t::value);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items : {1, 4})
|
||||
{
|
||||
auto reference_result =
|
||||
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * block_size_t::value, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, block_size_t{}, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("reduce/this_block Broadcasted", "[reduce][this_block]", integral_type_list, block_size_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = cuda::std::plus<>;
|
||||
using block_size_t = c2h::get<1, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * block_size_t::value);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items : {1, 4})
|
||||
{
|
||||
c2h::device_vector<value_t> d_out(block_size_t::value);
|
||||
auto reference_result =
|
||||
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * block_size_t::value, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, block_size_t{}, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
|
||||
verify_results(c2h::host_vector<value_t>(block_size_t::value, reference_result), c2h::host_vector<value_t>(d_out));
|
||||
}
|
||||
}
|
||||
@@ -1,242 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/functional>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/algorithm>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/coop.cuh>
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include <testing.cuh>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <c2h/extended_types.h>
|
||||
#include <c2h/generators.h>
|
||||
#include <catch2/matchers/catch_matchers_floating_point.hpp>
|
||||
|
||||
constexpr int block_size = 64;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Thread Reduce Wrapper Kernels
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <bool Broadcasted>
|
||||
struct ReduceKernel
|
||||
{
|
||||
template <class Config, int NumItems, class T, class RedOp>
|
||||
__device__ void operator()(
|
||||
Config config,
|
||||
cuda::std::integral_constant<int, NumItems>,
|
||||
const T* __restrict__ d_in,
|
||||
T* __restrict__ d_out,
|
||||
RedOp red_op)
|
||||
{
|
||||
cudax::this_cluster cluster{config};
|
||||
|
||||
T thread_data[NumItems];
|
||||
for (int i = 0; i < NumItems; ++i)
|
||||
{
|
||||
thread_data[i] = d_in[cuda::gpu_thread.rank_as<int>(cluster) + i * cuda::gpu_thread.count_as<int>(cluster)];
|
||||
}
|
||||
|
||||
if constexpr (Broadcasted)
|
||||
{
|
||||
const auto result = cudax::coop::reduce(cudax::broadcasted, cluster, thread_data, red_op);
|
||||
|
||||
d_out[cuda::gpu_thread.rank(cluster)] = result;
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto result = cudax::coop::reduce(cluster, thread_data, red_op);
|
||||
|
||||
REQUIRE(result.has_value() == cuda::gpu_thread.is_root_rank(cluster));
|
||||
if (cuda::gpu_thread.is_root_rank(cluster))
|
||||
{
|
||||
*d_out = result.value();
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Type list definition
|
||||
**********************************************************************************************************************/
|
||||
|
||||
using integral_type_list =
|
||||
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
|
||||
|
||||
using fp_type_list = c2h::type_list<float, double>;
|
||||
|
||||
using operator_integral_list =
|
||||
c2h::type_list<cuda::std::plus<>,
|
||||
cuda::std::multiplies<>,
|
||||
cuda::std::bit_and<>,
|
||||
cuda::std::bit_or<>,
|
||||
cuda::std::bit_xor<>,
|
||||
cuda::minimum<>,
|
||||
cuda::maximum<>>;
|
||||
|
||||
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
|
||||
|
||||
using cluster_size_list = c2h::enum_type_list<int, 1, 2, 4, 8>;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Verify results and kernel launch
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <class T>
|
||||
void verify_results(const T& expected_data, const T& test_results)
|
||||
{
|
||||
if constexpr (cuda::std::is_floating_point_v<T>)
|
||||
{
|
||||
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
|
||||
}
|
||||
else
|
||||
{
|
||||
REQUIRE(expected_data == test_results);
|
||||
}
|
||||
}
|
||||
|
||||
template <int ClusterSize, class T, class RedOp, bool Broadcasted = false>
|
||||
void run_reduce_kernel(
|
||||
cuda::stream_ref stream,
|
||||
cuda::std::integral_constant<int, ClusterSize>,
|
||||
int num_items,
|
||||
const c2h::device_vector<T>& in,
|
||||
c2h::device_vector<T>& out,
|
||||
RedOp red_op,
|
||||
cuda::std::bool_constant<Broadcasted> = {})
|
||||
{
|
||||
const auto config =
|
||||
cuda::make_config(cuda::grid_dims<1>(), cuda::cluster_dims<ClusterSize>(), cuda::block_dims<block_size>());
|
||||
const auto in_ptr = thrust::raw_pointer_cast(in.data());
|
||||
const auto out_ptr = thrust::raw_pointer_cast(out.data());
|
||||
const ReduceKernel<Broadcasted> kernel{};
|
||||
|
||||
switch (num_items)
|
||||
{
|
||||
case 1:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 1>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 4:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 4>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
default:
|
||||
FAIL("Unsupported number of items");
|
||||
}
|
||||
stream.sync();
|
||||
}
|
||||
|
||||
constexpr int max_size = 4;
|
||||
constexpr int num_seeds = 10;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Test cases
|
||||
**********************************************************************************************************************/
|
||||
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
|
||||
|
||||
C2H_TEST("reduce/this_cluster Integral Type Tests",
|
||||
"[reduce][this_cluster]",
|
||||
integral_type_list,
|
||||
operator_integral_list,
|
||||
cluster_size_list)
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
if (cuda::device_attributes::compute_capability_major(device) < 9)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
using cluster_size_t = c2h::get<2, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * cluster_size_t::value * block_size);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{device};
|
||||
for (int num_items : {1, 4})
|
||||
{
|
||||
auto reference_result = cuda::std::accumulate(
|
||||
h_in.begin(), h_in.begin() + num_items * cluster_size_t::value * block_size, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, cluster_size_t{}, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("reduce/this_cluster Floating-Point Type Tests",
|
||||
"[reduce][this_cluster]",
|
||||
fp_type_list,
|
||||
operator_fp_list,
|
||||
cluster_size_list)
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
if (cuda::device_attributes::compute_capability_major(device) < 9)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
using cluster_size_t = c2h::get<2, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
const auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * cluster_size_t::value * block_size);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{device};
|
||||
for (int num_items : {1, 4})
|
||||
{
|
||||
auto reference_result = cuda::std::accumulate(
|
||||
h_in.begin(), h_in.begin() + num_items * cluster_size_t::value * block_size, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, cluster_size_t{}, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("reduce/this_cluster Broadcasted", "[reduce][this_cluster]", integral_type_list, cluster_size_list)
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
if (cuda::device_attributes::compute_capability_major(device) < 9)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = cuda::std::plus<>;
|
||||
using cluster_size_t = c2h::get<1, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * cluster_size_t::value * block_size);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{device};
|
||||
for (int num_items : {1, 4})
|
||||
{
|
||||
c2h::device_vector<value_t> d_out(cluster_size_t::value * block_size);
|
||||
auto reference_result = cuda::std::accumulate(
|
||||
h_in.begin(), h_in.begin() + num_items * cluster_size_t::value * block_size, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, cluster_size_t{}, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
|
||||
verify_results(c2h::host_vector<value_t>(cluster_size_t::value * block_size, reference_result),
|
||||
c2h::host_vector<value_t>(d_out));
|
||||
}
|
||||
}
|
||||
@@ -1,252 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/functional>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/algorithm>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/coop.cuh>
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include <testing.cuh>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <c2h/extended_types.h>
|
||||
#include <c2h/generators.h>
|
||||
#include <catch2/matchers/catch_matchers_floating_point.hpp>
|
||||
|
||||
constexpr int cluster_size = 2;
|
||||
constexpr int block_size = 128;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Thread Reduce Wrapper Kernels
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <bool Broadcasted>
|
||||
struct ReduceKernel
|
||||
{
|
||||
template <class Config, int NumItems, class T, class RedOp>
|
||||
__device__ void operator()(
|
||||
Config config,
|
||||
cuda::std::integral_constant<int, NumItems>,
|
||||
const T* __restrict__ d_in,
|
||||
T* __restrict__ d_out,
|
||||
RedOp red_op)
|
||||
{
|
||||
cudax::this_grid grid{config};
|
||||
|
||||
T thread_data[NumItems];
|
||||
for (int i = 0; i < NumItems; ++i)
|
||||
{
|
||||
thread_data[i] = d_in[cuda::gpu_thread.rank_as<int>(grid) + i * cuda::gpu_thread.count_as<int>(grid)];
|
||||
}
|
||||
|
||||
if constexpr (Broadcasted)
|
||||
{
|
||||
const auto result = cudax::coop::reduce(cudax::broadcasted, grid, thread_data, red_op);
|
||||
|
||||
d_out[cuda::gpu_thread.rank(grid)] = result;
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto result = cudax::coop::reduce(grid, thread_data, red_op);
|
||||
|
||||
REQUIRE(result.has_value() == cuda::gpu_thread.is_root_rank(grid));
|
||||
if (cuda::gpu_thread.is_root_rank(grid))
|
||||
{
|
||||
*d_out = result.value();
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Type list definition
|
||||
**********************************************************************************************************************/
|
||||
|
||||
using integral_type_list =
|
||||
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
|
||||
|
||||
using fp_type_list = c2h::type_list<float, double>;
|
||||
|
||||
using operator_integral_list =
|
||||
c2h::type_list<cuda::std::plus<>,
|
||||
cuda::std::multiplies<>,
|
||||
cuda::std::bit_and<>,
|
||||
cuda::std::bit_or<>,
|
||||
cuda::std::bit_xor<>,
|
||||
cuda::minimum<>,
|
||||
cuda::maximum<>>;
|
||||
|
||||
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
|
||||
|
||||
using grid_size_list = c2h::enum_type_list<int, 1, 12, 32>;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Verify results and kernel launch`
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <class T>
|
||||
void verify_results(const T& expected_data, const T& test_results)
|
||||
{
|
||||
if constexpr (cuda::std::is_floating_point_v<T>)
|
||||
{
|
||||
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
|
||||
}
|
||||
else
|
||||
{
|
||||
REQUIRE(expected_data == test_results);
|
||||
}
|
||||
}
|
||||
|
||||
template <int GridSize, class T, class RedOp, bool Broadcasted = false>
|
||||
void run_reduce_kernel(
|
||||
cuda::stream_ref stream,
|
||||
cuda::std::integral_constant<int, GridSize>,
|
||||
int num_items,
|
||||
const c2h::device_vector<T>& in,
|
||||
c2h::device_vector<T>& out,
|
||||
RedOp red_op,
|
||||
cuda::std::bool_constant<Broadcasted> = {})
|
||||
{
|
||||
const auto config = cuda::make_config(
|
||||
cuda::grid_dims<GridSize>(),
|
||||
cuda::cluster_dims<cluster_size>(),
|
||||
cuda::block_dims<block_size>(),
|
||||
cuda::cooperative_launch{});
|
||||
const auto in_ptr = thrust::raw_pointer_cast(in.data());
|
||||
const auto out_ptr = thrust::raw_pointer_cast(out.data());
|
||||
const ReduceKernel<Broadcasted> kernel{};
|
||||
|
||||
switch (num_items)
|
||||
{
|
||||
case 1:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 1>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 4:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 4>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
default:
|
||||
FAIL("Unsupported number of items");
|
||||
}
|
||||
stream.sync();
|
||||
}
|
||||
|
||||
constexpr int max_size = 4;
|
||||
constexpr int num_seeds = 10;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Test cases
|
||||
**********************************************************************************************************************/
|
||||
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
|
||||
|
||||
C2H_TEST("reduce/this_grid Integral Type Tests",
|
||||
"[reduce][this_grid]",
|
||||
integral_type_list,
|
||||
operator_integral_list,
|
||||
grid_size_list)
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
if (cuda::device_attributes::compute_capability_major(device) < 9)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
using grid_size_t = c2h::get<2, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * grid_size_t::value * cluster_size * block_size);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{device};
|
||||
for (int num_items : {1, 4})
|
||||
{
|
||||
auto reference_result = cuda::std::accumulate(
|
||||
h_in.begin(),
|
||||
h_in.begin() + num_items * grid_size_t::value * cluster_size * block_size,
|
||||
operator_identity,
|
||||
reduce_op);
|
||||
run_reduce_kernel(stream, grid_size_t{}, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST(
|
||||
"reduce/this_grid Floating-Point Type Tests", "[reduce][this_grid]", fp_type_list, operator_fp_list, grid_size_list)
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
if (cuda::device_attributes::compute_capability_major(device) < 9)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
using grid_size_t = c2h::get<2, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
const auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * grid_size_t::value * cluster_size * block_size);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{device};
|
||||
for (int num_items : {1, 4})
|
||||
{
|
||||
auto reference_result = cuda::std::accumulate(
|
||||
h_in.begin(),
|
||||
h_in.begin() + num_items * grid_size_t::value * cluster_size * block_size,
|
||||
operator_identity,
|
||||
reduce_op);
|
||||
run_reduce_kernel(stream, grid_size_t{}, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("reduce/this_grid Broadcasted", "[reduce][this_grid]", integral_type_list, grid_size_list)
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
if (cuda::device_attributes::compute_capability_major(device) < 9)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = cuda::std::plus<>;
|
||||
using grid_size_t = c2h::get<1, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * grid_size_t::value * cluster_size * block_size);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{device};
|
||||
for (int num_items : {1, 4})
|
||||
{
|
||||
c2h::device_vector<value_t> d_out(grid_size_t::value * cluster_size * block_size);
|
||||
auto reference_result = cuda::std::accumulate(
|
||||
h_in.begin(),
|
||||
h_in.begin() + num_items * grid_size_t::value * cluster_size * block_size,
|
||||
operator_identity,
|
||||
reduce_op);
|
||||
run_reduce_kernel(stream, grid_size_t{}, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
|
||||
verify_results(c2h::host_vector<value_t>(grid_size_t::value * cluster_size * block_size, reference_result),
|
||||
c2h::host_vector<value_t>(d_out));
|
||||
}
|
||||
}
|
||||
@@ -1,242 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/functional>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/algorithm>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/coop.cuh>
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include <testing.cuh>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <c2h/extended_types.h>
|
||||
#include <c2h/generators.h>
|
||||
#include <catch2/matchers/catch_matchers_floating_point.hpp>
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Thread Reduce Wrapper Kernels
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <bool Broadcasted>
|
||||
struct ReduceKernel
|
||||
{
|
||||
template <class Config, int NumItems, class T, class RedOp>
|
||||
__device__ void operator()(
|
||||
Config config,
|
||||
cuda::std::integral_constant<int, NumItems>,
|
||||
const T* __restrict__ d_in,
|
||||
T* __restrict__ d_out,
|
||||
RedOp red_op)
|
||||
{
|
||||
cudax::this_thread thread{config};
|
||||
|
||||
T thread_data[NumItems];
|
||||
for (int i = 0; i < NumItems; ++i)
|
||||
{
|
||||
thread_data[i] = d_in[i];
|
||||
}
|
||||
|
||||
if constexpr (Broadcasted)
|
||||
{
|
||||
const auto result = cudax::coop::reduce(cudax::broadcasted, thread, thread_data, red_op);
|
||||
|
||||
*d_out = result;
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto result = cudax::coop::reduce(thread, thread_data, red_op);
|
||||
|
||||
REQUIRE(result.has_value());
|
||||
*d_out = result.value();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Type list definition
|
||||
**********************************************************************************************************************/
|
||||
|
||||
using integral_type_list =
|
||||
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
|
||||
|
||||
using fp_type_list = c2h::type_list<float, double>;
|
||||
|
||||
using operator_integral_list =
|
||||
c2h::type_list<cuda::std::plus<>,
|
||||
cuda::std::multiplies<>,
|
||||
cuda::std::bit_and<>,
|
||||
cuda::std::bit_or<>,
|
||||
cuda::std::bit_xor<>,
|
||||
cuda::minimum<>,
|
||||
cuda::maximum<>>;
|
||||
|
||||
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Verify results and kernel launch
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <class T>
|
||||
void verify_results(const T& expected_data, const T& test_results)
|
||||
{
|
||||
if constexpr (cuda::std::is_floating_point_v<T>)
|
||||
{
|
||||
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
|
||||
}
|
||||
else
|
||||
{
|
||||
REQUIRE(expected_data == test_results);
|
||||
}
|
||||
}
|
||||
|
||||
template <class T, class RedOp, bool Broadcasted = false>
|
||||
void run_reduce_kernel(
|
||||
cuda::stream_ref stream,
|
||||
int num_items,
|
||||
const c2h::device_vector<T>& in,
|
||||
c2h::device_vector<T>& out,
|
||||
RedOp red_op,
|
||||
cuda::std::bool_constant<Broadcasted> = {})
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<1>());
|
||||
const auto in_ptr = thrust::raw_pointer_cast(in.data());
|
||||
const auto out_ptr = thrust::raw_pointer_cast(out.data());
|
||||
const ReduceKernel<Broadcasted> kernel{};
|
||||
|
||||
switch (num_items)
|
||||
{
|
||||
case 1:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 1>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 2:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 2>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 3:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 3>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 4:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 4>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 5:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 5>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 6:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 6>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 7:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 7>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 8:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 8>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 9:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 9>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 10:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 10>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 11:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 11>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 12:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 12>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 13:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 13>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 14:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 14>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 15:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 15>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 16:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 16>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
default:
|
||||
FAIL("Unsupported number of items");
|
||||
}
|
||||
stream.sync();
|
||||
}
|
||||
|
||||
constexpr int max_size = 16;
|
||||
constexpr int num_seeds = 10;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Test cases
|
||||
**********************************************************************************************************************/
|
||||
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
|
||||
|
||||
C2H_TEST("reduce/this_thread Integral Type Tests", "[reduce][this_thread]", integral_type_list, operator_integral_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items = 1; num_items <= max_size; ++num_items)
|
||||
{
|
||||
auto reference_result = cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("reduce/this_thread Floating-Point Type Tests", "[reduce][this_thread]", fp_type_list, operator_fp_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
const auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items = 1; num_items <= max_size; ++num_items)
|
||||
{
|
||||
auto reference_result = cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("reduce/this_thread Broadcasted", "[reduce][this_thread]", integral_type_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = cuda::std::plus<>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items = 1; num_items <= max_size; ++num_items)
|
||||
{
|
||||
auto reference_result = cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
@@ -1,213 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/functional>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/algorithm>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/coop.cuh>
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include <testing.cuh>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <c2h/extended_types.h>
|
||||
#include <c2h/generators.h>
|
||||
#include <catch2/matchers/catch_matchers_floating_point.hpp>
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Thread Reduce Wrapper Kernels
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <bool Broadcasted>
|
||||
struct ReduceKernel
|
||||
{
|
||||
template <class Config, int NumItems, class T, class RedOp>
|
||||
__device__ void operator()(
|
||||
Config config,
|
||||
cuda::std::integral_constant<int, NumItems>,
|
||||
const T* __restrict__ d_in,
|
||||
T* __restrict__ d_out,
|
||||
RedOp red_op)
|
||||
{
|
||||
cudax::this_warp warp{config};
|
||||
|
||||
T thread_data[NumItems];
|
||||
for (int i = 0; i < NumItems; ++i)
|
||||
{
|
||||
thread_data[i] = d_in[cuda::gpu_thread.rank_as<int>(warp) + i * cuda::gpu_thread.count_as<int>(warp)];
|
||||
}
|
||||
|
||||
if constexpr (Broadcasted)
|
||||
{
|
||||
const auto result = cudax::coop::reduce(cudax::broadcasted, warp, thread_data, red_op);
|
||||
|
||||
d_out[cuda::gpu_thread.rank(warp)] = result;
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto result = cudax::coop::reduce(warp, thread_data, red_op);
|
||||
|
||||
REQUIRE(result.has_value() == cuda::gpu_thread.is_root_rank(warp));
|
||||
if (cuda::gpu_thread.is_root_rank(warp))
|
||||
{
|
||||
*d_out = result.value();
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Type list definition
|
||||
**********************************************************************************************************************/
|
||||
|
||||
using integral_type_list =
|
||||
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
|
||||
|
||||
using fp_type_list = c2h::type_list<float, double>;
|
||||
|
||||
using operator_integral_list =
|
||||
c2h::type_list<cuda::std::plus<>,
|
||||
cuda::std::multiplies<>,
|
||||
cuda::std::bit_and<>,
|
||||
cuda::std::bit_or<>,
|
||||
cuda::std::bit_xor<>,
|
||||
cuda::minimum<>,
|
||||
cuda::maximum<>>;
|
||||
|
||||
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Verify results and kernel launch
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <class T>
|
||||
void verify_results(const T& expected_data, const T& test_results)
|
||||
{
|
||||
if constexpr (cuda::std::is_floating_point_v<T>)
|
||||
{
|
||||
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
|
||||
}
|
||||
else
|
||||
{
|
||||
REQUIRE(expected_data == test_results);
|
||||
}
|
||||
}
|
||||
|
||||
template <class T, class RedOp, bool Broadcasted = false>
|
||||
void run_reduce_kernel(
|
||||
cuda::stream_ref stream,
|
||||
int num_items,
|
||||
const c2h::device_vector<T>& in,
|
||||
c2h::device_vector<T>& out,
|
||||
RedOp red_op,
|
||||
cuda::std::bool_constant<Broadcasted> = {})
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
|
||||
const auto in_ptr = thrust::raw_pointer_cast(in.data());
|
||||
const auto out_ptr = thrust::raw_pointer_cast(out.data());
|
||||
const ReduceKernel<Broadcasted> kernel{};
|
||||
|
||||
switch (num_items)
|
||||
{
|
||||
case 1:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 1>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 2:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 2>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 3:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 3>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 4:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 4>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
default:
|
||||
FAIL("Unsupported number of items");
|
||||
}
|
||||
stream.sync();
|
||||
}
|
||||
|
||||
constexpr int warp_size = 32;
|
||||
constexpr int max_size = 4;
|
||||
constexpr int num_seeds = 10;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Test cases
|
||||
**********************************************************************************************************************/
|
||||
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
|
||||
|
||||
C2H_TEST("reduce/this_warp Integral Type Tests", "[reduce][this_warp]", integral_type_list, operator_integral_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * warp_size);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items = 1; num_items <= max_size; ++num_items)
|
||||
{
|
||||
auto reference_result =
|
||||
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * warp_size, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("reduce/this_warp Floating-Point Type Tests", "[reduce][this_warp]", fp_type_list, operator_fp_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
const auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * warp_size);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items = 1; num_items <= max_size; ++num_items)
|
||||
{
|
||||
auto reference_result =
|
||||
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * warp_size, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("reduce/this_warp Broadcasted", "[reduce][this_warp]", integral_type_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = ::cuda::std::plus<>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * warp_size);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items = 1; num_items <= max_size; ++num_items)
|
||||
{
|
||||
c2h::device_vector<value_t> d_out(warp_size);
|
||||
auto reference_result =
|
||||
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * warp_size, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
|
||||
verify_results(c2h::host_vector<value_t>(warp_size, reference_result), c2h::host_vector<value_t>(d_out));
|
||||
}
|
||||
}
|
||||
@@ -1,279 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/atomic>
|
||||
#include <cuda/devices>
|
||||
#include <cuda/functional>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/algorithm>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/coop.cuh>
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include <testing.cuh>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <c2h/extended_types.h>
|
||||
#include <c2h/generators.h>
|
||||
#include <catch2/matchers/catch_matchers_floating_point.hpp>
|
||||
|
||||
constexpr int warp_size = 32;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Thread Reduce Wrapper Kernels
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <bool Broadcasted>
|
||||
struct ReduceKernel
|
||||
{
|
||||
template <class Config, unsigned NThreadsInGroup, int NumItems, class T, class RedOp>
|
||||
__device__ void operator()(
|
||||
Config config,
|
||||
cuda::std::integral_constant<unsigned, NThreadsInGroup>,
|
||||
cuda::std::integral_constant<int, NumItems>,
|
||||
const T* __restrict__ d_in,
|
||||
T* __restrict__ d_out,
|
||||
RedOp red_op)
|
||||
{
|
||||
cudax::group group{
|
||||
cuda::gpu_thread, cudax::this_warp{config}, cudax::group_by<NThreadsInGroup, false>{}, cudax::lane_synchronizer{}};
|
||||
|
||||
// All threads that are not part of the groups should exit early.
|
||||
if (!cuda::gpu_thread.is_part_of(group))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// We want to work only with one group, all other groups should exit early.
|
||||
if (group.rank(cuda::warp) > 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
T thread_data[NumItems];
|
||||
for (int i = 0; i < NumItems; ++i)
|
||||
{
|
||||
thread_data[i] = d_in[cuda::gpu_thread.rank_as<int>(group) + i * cuda::gpu_thread.count_as<int>(group)];
|
||||
}
|
||||
|
||||
if constexpr (Broadcasted)
|
||||
{
|
||||
const auto result = cudax::coop::reduce(cudax::broadcasted, group, thread_data, red_op);
|
||||
|
||||
d_out[cuda::gpu_thread.rank(group)] = result;
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto result = cudax::coop::reduce(group, thread_data, red_op);
|
||||
|
||||
REQUIRE(result.has_value() == cuda::gpu_thread.is_root_rank(group));
|
||||
if (cuda::gpu_thread.is_root_rank(group))
|
||||
{
|
||||
*d_out = result.value();
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Type list definition
|
||||
**********************************************************************************************************************/
|
||||
|
||||
using integral_type_list =
|
||||
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
|
||||
|
||||
using fp_type_list = c2h::type_list<float, double>;
|
||||
|
||||
using operator_integral_list =
|
||||
c2h::type_list<cuda::std::plus<>,
|
||||
cuda::std::multiplies<>,
|
||||
cuda::std::bit_and<>,
|
||||
cuda::std::bit_or<>,
|
||||
cuda::std::bit_xor<>,
|
||||
cuda::minimum<>,
|
||||
cuda::maximum<>>;
|
||||
|
||||
using nthreads_in_group_list = c2h::enum_type_list<unsigned, 1, 2, 12, 31, 32>;
|
||||
|
||||
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Verify results and kernel launch
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <class T>
|
||||
void verify_results(const T& expected_data, const T& test_results)
|
||||
{
|
||||
if constexpr (cuda::std::is_floating_point_v<T>)
|
||||
{
|
||||
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
|
||||
}
|
||||
else
|
||||
{
|
||||
REQUIRE(expected_data == test_results);
|
||||
}
|
||||
}
|
||||
|
||||
template <unsigned NThreadsInGroup, class T, class RedOp, bool Broadcasted = false>
|
||||
void run_reduce_kernel(
|
||||
cuda::stream_ref stream,
|
||||
cuda::std::integral_constant<unsigned, NThreadsInGroup>,
|
||||
int num_items,
|
||||
const c2h::device_vector<T>& in,
|
||||
c2h::device_vector<T>& out,
|
||||
RedOp red_op,
|
||||
cuda::std::bool_constant<Broadcasted> = {})
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<warp_size>());
|
||||
const auto in_ptr = thrust::raw_pointer_cast(in.data());
|
||||
const auto out_ptr = thrust::raw_pointer_cast(out.data());
|
||||
const ReduceKernel<Broadcasted> kernel{};
|
||||
|
||||
switch (num_items)
|
||||
{
|
||||
case 1:
|
||||
cuda::launch(
|
||||
stream,
|
||||
config,
|
||||
kernel,
|
||||
cuda::std::integral_constant<unsigned, NThreadsInGroup>{},
|
||||
cuda::std::integral_constant<int, 1>{},
|
||||
in_ptr,
|
||||
out_ptr,
|
||||
red_op);
|
||||
break;
|
||||
case 2:
|
||||
cuda::launch(
|
||||
stream,
|
||||
config,
|
||||
kernel,
|
||||
cuda::std::integral_constant<unsigned, NThreadsInGroup>{},
|
||||
cuda::std::integral_constant<int, 2>{},
|
||||
in_ptr,
|
||||
out_ptr,
|
||||
red_op);
|
||||
break;
|
||||
case 3:
|
||||
cuda::launch(
|
||||
stream,
|
||||
config,
|
||||
kernel,
|
||||
cuda::std::integral_constant<unsigned, NThreadsInGroup>{},
|
||||
cuda::std::integral_constant<int, 3>{},
|
||||
in_ptr,
|
||||
out_ptr,
|
||||
red_op);
|
||||
break;
|
||||
case 4:
|
||||
cuda::launch(
|
||||
stream,
|
||||
config,
|
||||
kernel,
|
||||
cuda::std::integral_constant<unsigned, NThreadsInGroup>{},
|
||||
cuda::std::integral_constant<int, 4>{},
|
||||
in_ptr,
|
||||
out_ptr,
|
||||
red_op);
|
||||
break;
|
||||
default:
|
||||
FAIL("Unsupported number of items");
|
||||
}
|
||||
stream.sync();
|
||||
}
|
||||
|
||||
constexpr int max_size = 4;
|
||||
constexpr int num_seeds = 10;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Test cases
|
||||
**********************************************************************************************************************/
|
||||
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
|
||||
|
||||
C2H_TEST("reduce/threads_within_warp Integral Type Tests",
|
||||
"[reduce][threads_within_warp]",
|
||||
integral_type_list,
|
||||
operator_integral_list,
|
||||
nthreads_in_group_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
using nthreads_in_group_t = c2h::get<2, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
constexpr auto nthreads_in_group = nthreads_in_group_t::value;
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * nthreads_in_group);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items = 1; num_items <= max_size; ++num_items)
|
||||
{
|
||||
auto reference_result =
|
||||
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * nthreads_in_group, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, nthreads_in_group_t{}, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("reduce/threads_within_warp Floating-Point Type Tests",
|
||||
"[reduce][threads_within_warp]",
|
||||
fp_type_list,
|
||||
operator_fp_list,
|
||||
nthreads_in_group_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
using nthreads_in_group_t = c2h::get<2, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto nthreads_in_group = nthreads_in_group_t::value;
|
||||
const auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * nthreads_in_group);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items = 1; num_items <= max_size; ++num_items)
|
||||
{
|
||||
auto reference_result =
|
||||
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * nthreads_in_group, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, nthreads_in_group_t{}, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST(
|
||||
"reduce/threads_within_warp Broadcasted", "[reduce][threads_within_warp]", integral_type_list, nthreads_in_group_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = cuda::std::plus<>;
|
||||
using nthreads_in_group_t = c2h::get<1, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
constexpr auto nthreads_in_group = nthreads_in_group_t::value;
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * nthreads_in_group);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items = 1; num_items <= max_size; ++num_items)
|
||||
{
|
||||
c2h::device_vector<value_t> d_out(nthreads_in_group);
|
||||
auto reference_result =
|
||||
cuda::std::accumulate(h_in.begin(), h_in.begin() + num_items * nthreads_in_group, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, nthreads_in_group_t{}, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
|
||||
verify_results(c2h::host_vector<value_t>(nthreads_in_group, reference_result), c2h::host_vector<value_t>(d_out));
|
||||
}
|
||||
}
|
||||
@@ -1,234 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/atomic>
|
||||
#include <cuda/devices>
|
||||
#include <cuda/functional>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/std/algorithm>
|
||||
#include <cuda/std/type_traits>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/coop.cuh>
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include <testing.cuh>
|
||||
|
||||
#include <c2h/catch2_test_helper.h>
|
||||
#include <c2h/extended_types.h>
|
||||
#include <c2h/generators.h>
|
||||
#include <catch2/matchers/catch_matchers_floating_point.hpp>
|
||||
|
||||
constexpr int nwarps_in_group = 3;
|
||||
constexpr int warp_size = 32;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Thread Reduce Wrapper Kernels
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <bool Broadcasted>
|
||||
struct ReduceKernel
|
||||
{
|
||||
template <class Config, int NumItems, class T, class RedOp>
|
||||
__device__ void operator()(
|
||||
Config config,
|
||||
cuda::std::integral_constant<int, NumItems>,
|
||||
const T* __restrict__ d_in,
|
||||
T* __restrict__ d_out,
|
||||
RedOp red_op)
|
||||
{
|
||||
cudax::this_block block{config};
|
||||
|
||||
using Barriers = cuda::barrier<cuda::thread_scope_block>[1];
|
||||
__shared__ cuda::std::aligned_storage_t<sizeof(Barriers), alignof(Barriers)> barriers_storage;
|
||||
auto& barriers = reinterpret_cast<Barriers&>(barriers_storage);
|
||||
|
||||
cudax::group group{
|
||||
cuda::warp, block, cudax::group_by<nwarps_in_group, false>{}, cudax::barrier_synchronizer{barriers}};
|
||||
|
||||
// All threads that are not part of the groups should exit early.
|
||||
if (!cuda::gpu_thread.is_part_of(group))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
T thread_data[NumItems];
|
||||
for (int i = 0; i < NumItems; ++i)
|
||||
{
|
||||
thread_data[i] = d_in[cuda::gpu_thread.rank_as<int>(group) + i * cuda::gpu_thread.count_as<int>(group)];
|
||||
}
|
||||
|
||||
if constexpr (Broadcasted)
|
||||
{
|
||||
const auto result = cudax::coop::reduce(cudax::broadcasted, group, thread_data, red_op);
|
||||
|
||||
d_out[cuda::gpu_thread.rank(group)] = result;
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto result = cudax::coop::reduce(group, thread_data, red_op);
|
||||
|
||||
REQUIRE(result.has_value() == cuda::gpu_thread.is_root_rank(group));
|
||||
if (cuda::gpu_thread.is_root_rank(group))
|
||||
{
|
||||
*d_out = result.value();
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Type list definition
|
||||
**********************************************************************************************************************/
|
||||
|
||||
using integral_type_list =
|
||||
c2h::type_list<cuda::std::int8_t, cuda::std::int16_t, cuda::std::uint16_t, cuda::std::int32_t, cuda::std::int64_t>;
|
||||
|
||||
using fp_type_list = c2h::type_list<float, double>;
|
||||
|
||||
using operator_integral_list =
|
||||
c2h::type_list<cuda::std::plus<>,
|
||||
cuda::std::multiplies<>,
|
||||
cuda::std::bit_and<>,
|
||||
cuda::std::bit_or<>,
|
||||
cuda::std::bit_xor<>,
|
||||
cuda::minimum<>,
|
||||
cuda::maximum<>>;
|
||||
|
||||
using operator_fp_list = c2h::type_list<cuda::std::plus<>, cuda::std::multiplies<>, cuda::minimum<>, cuda::maximum<>>;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Verify results and kernel launch
|
||||
**********************************************************************************************************************/
|
||||
|
||||
template <class T>
|
||||
void verify_results(const T& expected_data, const T& test_results)
|
||||
{
|
||||
if constexpr (cuda::std::is_floating_point_v<T>)
|
||||
{
|
||||
REQUIRE_THAT(expected_data, Catch::Matchers::WithinRel(test_results, T{0.05}));
|
||||
}
|
||||
else
|
||||
{
|
||||
REQUIRE(expected_data == test_results);
|
||||
}
|
||||
}
|
||||
|
||||
template <class T, class RedOp, bool Broadcasted = false>
|
||||
void run_reduce_kernel(
|
||||
cuda::stream_ref stream,
|
||||
int num_items,
|
||||
const c2h::device_vector<T>& in,
|
||||
c2h::device_vector<T>& out,
|
||||
RedOp red_op,
|
||||
cuda::std::bool_constant<Broadcasted> = {})
|
||||
{
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<(nwarps_in_group + 2) * warp_size>());
|
||||
const auto in_ptr = thrust::raw_pointer_cast(in.data());
|
||||
const auto out_ptr = thrust::raw_pointer_cast(out.data());
|
||||
const ReduceKernel<Broadcasted> kernel{};
|
||||
|
||||
switch (num_items)
|
||||
{
|
||||
case 1:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 1>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 2:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 2>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 3:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 3>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
case 4:
|
||||
cuda::launch(stream, config, kernel, cuda::std::integral_constant<int, 4>{}, in_ptr, out_ptr, red_op);
|
||||
break;
|
||||
default:
|
||||
FAIL("Unsupported number of items");
|
||||
}
|
||||
stream.sync();
|
||||
}
|
||||
|
||||
constexpr int max_size = 4;
|
||||
constexpr int num_seeds = 10;
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Test cases
|
||||
**********************************************************************************************************************/
|
||||
|
||||
_CCCL_DIAG_SUPPRESS_MSVC(4244) // warning C4244: '=': conversion from 'int' to '_Tp', possible loss of data
|
||||
|
||||
C2H_TEST("reduce/warps_within_block Integral Type Tests",
|
||||
"[reduce][warps_within_block]",
|
||||
integral_type_list,
|
||||
operator_integral_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * nwarps_in_group * warp_size);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items = 1; num_items <= max_size; ++num_items)
|
||||
{
|
||||
auto reference_result = cuda::std::accumulate(
|
||||
h_in.begin(), h_in.begin() + num_items * nwarps_in_group * warp_size, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST(
|
||||
"reduce/warps_within_block Floating-Point Type Tests", "[reduce][warps_within_block]", fp_type_list, operator_fp_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = c2h::get<1, TestType>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
const auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * nwarps_in_group * warp_size);
|
||||
c2h::device_vector<value_t> d_out(1);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items = 1; num_items <= max_size; ++num_items)
|
||||
{
|
||||
auto reference_result = cuda::std::accumulate(
|
||||
h_in.begin(), h_in.begin() + num_items * nwarps_in_group * warp_size, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op);
|
||||
verify_results(reference_result, c2h::host_vector<value_t>(d_out)[0]);
|
||||
}
|
||||
}
|
||||
|
||||
C2H_TEST("reduce/warps_within_block Broadcasted", "[reduce][warps_within_block]", integral_type_list)
|
||||
{
|
||||
using value_t = c2h::get<0, TestType>;
|
||||
using op_t = cuda::std::plus<>;
|
||||
constexpr auto reduce_op = op_t{};
|
||||
constexpr auto operator_identity = cuda::identity_element<op_t, value_t>();
|
||||
CAPTURE(c2h::type_name<value_t>(), max_size, c2h::type_name<decltype(reduce_op)>());
|
||||
c2h::device_vector<value_t> d_in(max_size * nwarps_in_group * warp_size);
|
||||
c2h::gen(C2H_SEED(num_seeds), d_in, cuda::std::numeric_limits<value_t>::min());
|
||||
c2h::host_vector<value_t> h_in = d_in;
|
||||
cuda::stream stream{cuda::devices[0]};
|
||||
for (int num_items = 1; num_items <= max_size; ++num_items)
|
||||
{
|
||||
c2h::device_vector<value_t> d_out(nwarps_in_group * warp_size);
|
||||
auto reference_result = cuda::std::accumulate(
|
||||
h_in.begin(), h_in.begin() + num_items * nwarps_in_group * warp_size, operator_identity, reduce_op);
|
||||
run_reduce_kernel(stream, num_items, d_in, d_out, reduce_op, cuda::std::true_type{});
|
||||
verify_results(c2h::host_vector<value_t>(nwarps_in_group * warp_size, reference_result),
|
||||
c2h::host_vector<value_t>(d_out));
|
||||
}
|
||||
}
|
||||
@@ -1,141 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/stream>
|
||||
#include <cuda/type_traits>
|
||||
|
||||
#include <cuda/experimental/coop.cuh>
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "testing.cuh"
|
||||
|
||||
template <class T>
|
||||
__device__ T make_instance_for(unsigned rank)
|
||||
{
|
||||
if constexpr (cuda::is_vector_type_v<T>)
|
||||
{
|
||||
using U = cuda::scalar_type_t<T>;
|
||||
return T{static_cast<U>(rank), static_cast<U>(rank), static_cast<U>(rank)};
|
||||
}
|
||||
else
|
||||
{
|
||||
return static_cast<T>(rank);
|
||||
}
|
||||
}
|
||||
|
||||
template <class T, class Group>
|
||||
__device__ void test_group(const Group& group)
|
||||
{
|
||||
// Exit all threads that are not part of the group.
|
||||
if (!cuda::gpu_thread.is_part_of(group))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const auto my_rank = cuda::gpu_thread.rank_as<unsigned>(group);
|
||||
const auto my_value = make_instance_for<T>(my_rank);
|
||||
|
||||
// Test identity.
|
||||
REQUIRE(cudax::coop::shuffle(group, my_value, my_rank) == my_value);
|
||||
|
||||
// Test broadcast from root rank.
|
||||
REQUIRE(cudax::coop::shuffle(group, my_value, 0) == make_instance_for<T>(0));
|
||||
|
||||
// Test broadcast from last rank.
|
||||
{
|
||||
const auto other_rank = cuda::gpu_thread.count_as<unsigned>(group) - 1;
|
||||
REQUIRE(cudax::coop::shuffle(group, my_value, other_rank) == make_instance_for<T>(other_rank));
|
||||
}
|
||||
|
||||
// Test my_rank + 1.
|
||||
{
|
||||
const auto other_rank = (my_rank + 1) % cuda::gpu_thread.count_as<unsigned>(group);
|
||||
REQUIRE(cudax::coop::shuffle(group, my_value, other_rank) == make_instance_for<T>(other_rank));
|
||||
}
|
||||
|
||||
// Test my_rank + 6.
|
||||
{
|
||||
const auto other_rank = (my_rank + 6) % cuda::gpu_thread.count_as<unsigned>(group);
|
||||
REQUIRE(cudax::coop::shuffle(group, my_value, other_rank) == make_instance_for<T>(other_rank));
|
||||
}
|
||||
|
||||
// Test my_rank +- 1 based on whether my_rank is even or not.
|
||||
{
|
||||
const auto other_rank =
|
||||
((my_rank % 2 == 0) ? my_rank + 1 : my_rank - 1) % cuda::gpu_thread.count_as<unsigned>(group);
|
||||
REQUIRE(cudax::coop::shuffle(group, my_value, other_rank) == make_instance_for<T>(other_rank));
|
||||
}
|
||||
|
||||
// Test my_rank +- 3 based on whether my_rank is even or not.
|
||||
{
|
||||
const auto other_rank =
|
||||
((my_rank % 2 == 0) ? my_rank + 3 : my_rank - 3) % cuda::gpu_thread.count_as<unsigned>(group);
|
||||
REQUIRE(cudax::coop::shuffle(group, my_value, other_rank) == make_instance_for<T>(other_rank));
|
||||
}
|
||||
}
|
||||
|
||||
struct CustomBinaryPartition
|
||||
{
|
||||
template <class MappingResult>
|
||||
__device__ bool operator()(MappingResult mapping_result)
|
||||
{
|
||||
switch (mapping_result.unit_rank())
|
||||
{
|
||||
case 1:
|
||||
case 5:
|
||||
case 14:
|
||||
case 15:
|
||||
case 31:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class T, class Config>
|
||||
__device__ void test_type(const Config& config)
|
||||
{
|
||||
const cudax::this_warp warp{config};
|
||||
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::identity_mapping{}, cudax::lane_synchronizer{}});
|
||||
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::group_by<4>{}, cudax::lane_synchronizer{}});
|
||||
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::group_by{1}, cudax::lane_synchronizer{}});
|
||||
test_group<T>(
|
||||
cudax::group{cuda::gpu_thread, warp, cudax::group_by{3, cudax::non_exhaustive}, cudax::lane_synchronizer{}});
|
||||
test_group<T>(
|
||||
cudax::group{cuda::gpu_thread, warp, cudax::binary_partition{CustomBinaryPartition{}}, cudax::lane_synchronizer{}});
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_type<signed char>(config);
|
||||
test_type<unsigned>(config);
|
||||
test_type<unsigned long long>(config);
|
||||
test_type<longlong3>(config);
|
||||
}
|
||||
};
|
||||
|
||||
C2H_TEST("shuffle/threads_within_warp", "[shuffle][threads_within_warp]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,143 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/stream>
|
||||
#include <cuda/type_traits>
|
||||
|
||||
#include <cuda/experimental/coop.cuh>
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "testing.cuh"
|
||||
|
||||
template <class T>
|
||||
__device__ T make_instance_for(unsigned rank)
|
||||
{
|
||||
if constexpr (cuda::is_vector_type_v<T>)
|
||||
{
|
||||
using U = cuda::scalar_type_t<T>;
|
||||
return T{static_cast<U>(rank), static_cast<U>(rank), static_cast<U>(rank)};
|
||||
}
|
||||
else
|
||||
{
|
||||
return static_cast<T>(rank);
|
||||
}
|
||||
}
|
||||
|
||||
template <class T, class Group>
|
||||
__device__ void test_group(const Group& group)
|
||||
{
|
||||
// Exit all threads that are not part of the group.
|
||||
if (!cuda::gpu_thread.is_part_of(group))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const auto my_rank = cuda::gpu_thread.rank_as<unsigned>(group);
|
||||
const auto my_value = make_instance_for<T>(my_rank);
|
||||
|
||||
// Test identity.
|
||||
REQUIRE(cudax::coop::shuffle_down(group, my_value, 0) == my_value);
|
||||
|
||||
// Test getting value from the next rank.
|
||||
{
|
||||
const auto offset = 1;
|
||||
const auto other_rank = cuda::gpu_thread.rank(group) + offset;
|
||||
const auto ref = (other_rank < cuda::gpu_thread.count(group))
|
||||
? cuda::std::optional{make_instance_for<T>(other_rank)}
|
||||
: cuda::std::nullopt;
|
||||
REQUIRE(cudax::coop::shuffle_down(group, my_value, offset) == ref);
|
||||
}
|
||||
|
||||
// Test getting value from the this + 2 rank.
|
||||
{
|
||||
const auto offset = 2;
|
||||
const auto other_rank = cuda::gpu_thread.rank(group) + offset;
|
||||
const auto ref = (other_rank < cuda::gpu_thread.count(group))
|
||||
? cuda::std::optional{make_instance_for<T>(other_rank)}
|
||||
: cuda::std::nullopt;
|
||||
REQUIRE(cudax::coop::shuffle_down(group, my_value, offset) == ref);
|
||||
}
|
||||
|
||||
// Test getting value from the last rank.
|
||||
{
|
||||
const auto other_rank = cuda::gpu_thread.count(group) - 1;
|
||||
const auto offset = other_rank - cuda::gpu_thread.rank(group);
|
||||
REQUIRE(cudax::coop::shuffle_down(group, my_value, offset) == make_instance_for<T>(other_rank));
|
||||
}
|
||||
|
||||
// Test getting value from the last rank + 1.
|
||||
{
|
||||
const auto other_rank = cuda::gpu_thread.count(group);
|
||||
const auto offset = other_rank - cuda::gpu_thread.rank(group);
|
||||
REQUIRE(cudax::coop::shuffle_down(group, my_value, offset) == cuda::std::nullopt);
|
||||
}
|
||||
|
||||
// Test getting value from out of range offset.
|
||||
REQUIRE(cudax::coop::shuffle_down(group, my_value, ~0u) == cuda::std::nullopt);
|
||||
}
|
||||
|
||||
struct CustomBinaryPartition
|
||||
{
|
||||
template <class MappingResult>
|
||||
__device__ bool operator()(MappingResult mapping_result)
|
||||
{
|
||||
switch (mapping_result.unit_rank())
|
||||
{
|
||||
case 1:
|
||||
case 5:
|
||||
case 14:
|
||||
case 15:
|
||||
case 31:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class T, class Config>
|
||||
__device__ void test_type(const Config& config)
|
||||
{
|
||||
const cudax::this_warp warp{config};
|
||||
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::identity_mapping{}, cudax::lane_synchronizer{}});
|
||||
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::group_by<4>{}, cudax::lane_synchronizer{}});
|
||||
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::group_by{1}, cudax::lane_synchronizer{}});
|
||||
test_group<T>(
|
||||
cudax::group{cuda::gpu_thread, warp, cudax::group_by{3, cudax::non_exhaustive}, cudax::lane_synchronizer{}});
|
||||
test_group<T>(
|
||||
cudax::group{cuda::gpu_thread, warp, cudax::binary_partition{CustomBinaryPartition{}}, cudax::lane_synchronizer{}});
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_type<signed char>(config);
|
||||
test_type<unsigned>(config);
|
||||
test_type<unsigned long long>(config);
|
||||
test_type<longlong3>(config);
|
||||
}
|
||||
};
|
||||
|
||||
C2H_TEST("shuffle/threads_within_warp", "[shuffle][threads_within_warp]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
@@ -1,142 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <cuda/devices>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/launch>
|
||||
#include <cuda/stream>
|
||||
#include <cuda/type_traits>
|
||||
|
||||
#include <cuda/experimental/coop.cuh>
|
||||
#include <cuda/experimental/group.cuh>
|
||||
|
||||
#include "testing.cuh"
|
||||
|
||||
template <class T>
|
||||
__device__ T make_instance_for(unsigned rank)
|
||||
{
|
||||
if constexpr (cuda::is_vector_type_v<T>)
|
||||
{
|
||||
using U = cuda::scalar_type_t<T>;
|
||||
return T{static_cast<U>(rank), static_cast<U>(rank), static_cast<U>(rank)};
|
||||
}
|
||||
else
|
||||
{
|
||||
return static_cast<T>(rank);
|
||||
}
|
||||
}
|
||||
|
||||
template <class T, class Group>
|
||||
__device__ void test_group(const Group& group)
|
||||
{
|
||||
// Exit all threads that are not part of the group.
|
||||
if (!cuda::gpu_thread.is_part_of(group))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const auto my_rank = cuda::gpu_thread.rank_as<unsigned>(group);
|
||||
const auto my_value = make_instance_for<T>(my_rank);
|
||||
|
||||
// Test identity.
|
||||
REQUIRE(cudax::coop::shuffle_up(group, my_value, 0) == my_value);
|
||||
|
||||
// Test getting value from the previous rank.
|
||||
{
|
||||
const auto offset = 1;
|
||||
const auto other_rank = cuda::gpu_thread.rank(group) - offset;
|
||||
const auto ref = (other_rank < cuda::gpu_thread.count(group))
|
||||
? cuda::std::optional{make_instance_for<T>(other_rank)}
|
||||
: cuda::std::nullopt;
|
||||
REQUIRE(cudax::coop::shuffle_up(group, my_value, offset) == ref);
|
||||
}
|
||||
|
||||
// Test getting value from the this + 2 rank.
|
||||
{
|
||||
const auto offset = 2;
|
||||
const auto other_rank = cuda::gpu_thread.rank(group) - offset;
|
||||
const auto ref = (other_rank < cuda::gpu_thread.count(group))
|
||||
? cuda::std::optional{make_instance_for<T>(other_rank)}
|
||||
: cuda::std::nullopt;
|
||||
REQUIRE(cudax::coop::shuffle_up(group, my_value, offset) == ref);
|
||||
}
|
||||
|
||||
// Test getting value from the first rank.
|
||||
{
|
||||
const auto other_rank = 0;
|
||||
const auto offset = cuda::gpu_thread.rank(group) - other_rank;
|
||||
REQUIRE(cudax::coop::shuffle_up(group, my_value, offset) == make_instance_for<T>(other_rank));
|
||||
}
|
||||
|
||||
// Test getting value from the first rank - 1.
|
||||
{
|
||||
const auto offset = cuda::gpu_thread.rank(group) + 1;
|
||||
REQUIRE(cudax::coop::shuffle_up(group, my_value, offset) == cuda::std::nullopt);
|
||||
}
|
||||
|
||||
// Test getting value from out of range offset.
|
||||
REQUIRE(cudax::coop::shuffle_up(group, my_value, ~0u) == cuda::std::nullopt);
|
||||
}
|
||||
|
||||
struct CustomBinaryPartition
|
||||
{
|
||||
template <class MappingResult>
|
||||
__device__ bool operator()(MappingResult mapping_result)
|
||||
{
|
||||
switch (mapping_result.unit_rank())
|
||||
{
|
||||
case 1:
|
||||
case 5:
|
||||
case 14:
|
||||
case 15:
|
||||
case 31:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <class T, class Config>
|
||||
__device__ void test_type(const Config& config)
|
||||
{
|
||||
const cudax::this_warp warp{config};
|
||||
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::identity_mapping{}, cudax::lane_synchronizer{}});
|
||||
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::group_by<4>{}, cudax::lane_synchronizer{}});
|
||||
test_group<T>(cudax::group{cuda::gpu_thread, warp, cudax::group_by{1}, cudax::lane_synchronizer{}});
|
||||
test_group<T>(
|
||||
cudax::group{cuda::gpu_thread, warp, cudax::group_by{3, cudax::non_exhaustive}, cudax::lane_synchronizer{}});
|
||||
test_group<T>(
|
||||
cudax::group{cuda::gpu_thread, warp, cudax::binary_partition{CustomBinaryPartition{}}, cudax::lane_synchronizer{}});
|
||||
}
|
||||
|
||||
struct TestKernel
|
||||
{
|
||||
template <class Config>
|
||||
__device__ void operator()(const Config& config)
|
||||
{
|
||||
test_type<signed char>(config);
|
||||
test_type<unsigned>(config);
|
||||
test_type<unsigned long long>(config);
|
||||
test_type<longlong3>(config);
|
||||
}
|
||||
};
|
||||
|
||||
C2H_TEST("shuffle/threads_within_warp", "[shuffle][threads_within_warp]")
|
||||
{
|
||||
const auto device = cuda::devices[0];
|
||||
|
||||
const cuda::stream stream{device};
|
||||
|
||||
const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
|
||||
cuda::launch(stream, config, TestKernel{});
|
||||
|
||||
stream.sync();
|
||||
}
|
||||
Reference in New Issue
Block a user