[CCCL] 瘦身 + 补全: 移除 cudax/python/libcudacxx-tests 冗余文件, 新增 c2h 测试助手 + cmake 构建系统 + 8 个 CUDA thrust examples
变更摘要:
- 删除: cudax/ (783 files, 7.2M) — 实验性组件，竞赛不需要
- 删除: python/ (226 files, 2.0M) — Python 绑定，竞赛不需要
- 删除: libcudacxx/{test,benchmarks,codegen,cmake,share} (4432 files, 31M)
保留: libcudacxx/include/ (1463 headers, cuda::std 编译依赖)
- 新增: c2h/ (27 files) — CUB Catch2 测试辅助头文件，编译 243 个测试必需
- 新增: cmake/ (29 files) — CCCL 原生 CMake 构建系统
- 新增: thrust/examples/cuda/ (7 files) + cpp_integration/ (1 file)
async_reduce, custom_temporary_allocation, explicit_cuda_stream,
global_device_vector, range_view, unwrap_pointer, wrap_pointer, device
结果: cccl_upstream 从 74M→35M (瘦身 53%), 核心内容 100% 保留:
27/27 tuning headers, 78 benchmarks, 243 tests,
60 thrust examples, 18 CUB examples, 全部编译头文件
This commit is contained in:
muh-bot
@@ -1,68 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_CONCEPTS_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_CONCEPTS_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__fwd/hierarchy.h>
|
||||
#include <cuda/__warp/lane_mask.h>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__concepts/same_as.h>
|
||||
#include <cuda/std/__type_traits/is_copy_constructible.h>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
template <class _Group>
|
||||
_CCCL_CONCEPT is_group = _CCCL_REQUIRES_EXPR((_Group), _Group&& __g, const _Group&& __cg)(
|
||||
typename(typename _Group::unit_type),
|
||||
requires(__is_hierarchy_level_v<typename _Group::unit_type>),
|
||||
typename(typename _Group::level_type),
|
||||
requires(__is_hierarchy_level_v<typename _Group::level_type>),
|
||||
typename(typename _Group::hierarchy_type),
|
||||
requires(__is_hierarchy_v<typename _Group::hierarchy_type>),
|
||||
_Same_as(void) __g.sync(),
|
||||
_Same_as(void) __g.sync_aligned(),
|
||||
_Same_as(const typename _Group::hierarchy_type&) __cg.hierarchy()
|
||||
// todo: add __sub_unit_queryable and __super_unit_queryable
|
||||
);
|
||||
|
||||
template <class _Tp>
|
||||
_CCCL_CONCEPT __group_mapping_result = _CCCL_REQUIRES_EXPR((_Tp), const _Tp& __v)(
|
||||
requires(::cuda::std::is_copy_constructible_v<_Tp>),
|
||||
_Same_as(::cuda::std::size_t) _Tp::static_group_count(),
|
||||
_Same_as(unsigned) __v.group_count(),
|
||||
_Same_as(unsigned) __v.group_rank(),
|
||||
_Same_as(::cuda::std::size_t) _Tp::static_unit_count(),
|
||||
_Same_as(unsigned) __v.unit_count(),
|
||||
_Same_as(unsigned) __v.unit_rank(),
|
||||
_Same_as(::cuda::device::lane_mask) __v.lane_mask(),
|
||||
_Same_as(bool) _Tp::is_always_exhaustive(),
|
||||
_Same_as(bool) _Tp::is_always_contiguous());
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_CONCEPTS_CUH
|
||||
@@ -1,128 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_FWD_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_FWD_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
// Q: Do we want to enable this by default, or do we want the user to define some macro to get the interoperability with
|
||||
// cooperative groups?
|
||||
#if __has_include(<cooperative_groups.h>)
|
||||
# define _CCCL_HAS_COOPERATIVE_GROUPS() 1
|
||||
#else // ^^^ has cooperative groups ^^^ / vvv no cooperative groups vvv
|
||||
# define _CCCL_HAS_COOPERATIVE_GROUPS() 0
|
||||
#endif // ^^^ no cooperative groups ^^^
|
||||
|
||||
#include <cuda/__fwd/hierarchy.h>
|
||||
#include <cuda/std/__cstddef/types.h>
|
||||
#include <cuda/std/__fwd/extents.h>
|
||||
#include <cuda/std/__fwd/span.h>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
using __implicit_hierarchy_t =
|
||||
hierarchy<thread_level,
|
||||
hierarchy_level_desc<grid_level, ::cuda::std::dims<3, unsigned>>,
|
||||
hierarchy_level_desc<cluster_level, ::cuda::std::dims<3, unsigned>>,
|
||||
hierarchy_level_desc<block_level, ::cuda::std::dims<3, unsigned>>>;
|
||||
|
||||
// groups
|
||||
|
||||
template <class _Level, class _Hierarchy>
|
||||
class __this_group_base;
|
||||
|
||||
template <class _Hierarchy>
|
||||
class this_thread;
|
||||
|
||||
template <class _Hierarchy>
|
||||
class this_warp;
|
||||
|
||||
template <class _Hierarchy>
|
||||
class this_block;
|
||||
|
||||
template <class _Hierarchy>
|
||||
class this_cluster;
|
||||
|
||||
template <class _Hierarchy>
|
||||
class this_grid;
|
||||
|
||||
template <class _Unit, class _ParentGroup, class _Mapping, class _Synchronizer>
|
||||
class group;
|
||||
|
||||
// mappings
|
||||
|
||||
template <class _Fn>
|
||||
class binary_partition;
|
||||
|
||||
template <::cuda::std::size_t _UnitCount = ::cuda::std::dynamic_extent, bool _IsExhaustive = true>
|
||||
class group_by;
|
||||
|
||||
template <class _Data, bool _IsExahustive>
|
||||
class group_as;
|
||||
|
||||
class identity_mapping;
|
||||
|
||||
template <::cuda::std::size_t _UnitCount = ::cuda::std::dynamic_extent>
|
||||
class take;
|
||||
|
||||
// synchronizers
|
||||
|
||||
class lane_synchronizer;
|
||||
|
||||
template <class _Barrier, ::cuda::std::size_t _Np>
|
||||
class barrier_synchronizer;
|
||||
|
||||
// traits
|
||||
|
||||
template <class _Tp>
|
||||
inline constexpr bool __is_this_group_v = false;
|
||||
template <class _Hierarchy>
|
||||
inline constexpr bool __is_this_group_v<this_thread<_Hierarchy>> = true;
|
||||
template <class _Hierarchy>
|
||||
inline constexpr bool __is_this_group_v<this_warp<_Hierarchy>> = true;
|
||||
template <class _Hierarchy>
|
||||
inline constexpr bool __is_this_group_v<this_block<_Hierarchy>> = true;
|
||||
template <class _Hierarchy>
|
||||
inline constexpr bool __is_this_group_v<this_cluster<_Hierarchy>> = true;
|
||||
template <class _Hierarchy>
|
||||
inline constexpr bool __is_this_group_v<this_grid<_Hierarchy>> = true;
|
||||
|
||||
template <class _Tp>
|
||||
inline constexpr bool __is_group_mapping_v = false;
|
||||
template <::cuda::std::size_t _UnitCount, bool _IsExhaustive>
|
||||
inline constexpr bool __is_group_mapping_v<group_by<_UnitCount, _IsExhaustive>> = true;
|
||||
template <class _Data, bool _IsExhaustive>
|
||||
inline constexpr bool __is_group_mapping_v<group_as<_Data, _IsExhaustive>> = true;
|
||||
|
||||
// tags
|
||||
|
||||
struct non_exhaustive_t;
|
||||
|
||||
inline constexpr unsigned __invalid_count_or_rank = 0xffff'ffff;
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_FWD_CUH
|
||||
@@ -1,263 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_GROUP_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_GROUP_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__bit/bitmask.h>
|
||||
#include <cuda/__cmath/pow2.h>
|
||||
#include <cuda/__hierarchy/queries/count.h>
|
||||
#include <cuda/__hierarchy/queries/rank.h>
|
||||
#include <cuda/barrier>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/std/__bit/popcount.h>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__limits/numeric_limits.h>
|
||||
#include <cuda/std/__type_traits/is_constructible.h>
|
||||
#include <cuda/std/__type_traits/is_integer.h>
|
||||
#include <cuda/std/__type_traits/is_same.h>
|
||||
#include <cuda/std/span>
|
||||
|
||||
#include <cuda/experimental/__group/concepts.cuh>
|
||||
#include <cuda/experimental/__group/fwd.cuh>
|
||||
#include <cuda/experimental/__group/mapping/group_by.cuh>
|
||||
#include <cuda/experimental/__group/mapping/mapping_result.cuh>
|
||||
#include <cuda/experimental/__group/this_group.cuh>
|
||||
#include <cuda/experimental/__group/traits.cuh>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
template <class _Unit, class _ParentGroup, class _Mapping, class _Synchronizer>
|
||||
class group
|
||||
{
|
||||
static_assert(__is_hierarchy_level_v<_Unit>);
|
||||
static_assert(is_group<_ParentGroup>);
|
||||
|
||||
// todo(dabayer): Allow groups stacking and remove this.
|
||||
static_assert(__is_this_group_v<_ParentGroup>);
|
||||
|
||||
// todo(dabayer): static_assert that _Unit is (under) typename _ParentGroup::unit_type
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr auto
|
||||
__get_initial_mapping_result(const _ParentGroup& __parent) noexcept
|
||||
{
|
||||
using _ParentMappingResult = typename _ParentGroup::__mapping_result_type;
|
||||
using _MappingResult =
|
||||
::cuda::experimental::__mapping_result<1,
|
||||
::cuda::experimental::__static_count_query_group<_Unit, _ParentGroup>(),
|
||||
_ParentMappingResult::is_always_exhaustive(),
|
||||
_ParentMappingResult::is_always_contiguous()>;
|
||||
return _MappingResult{
|
||||
1,
|
||||
0,
|
||||
::cuda::experimental::__count_query_group<unsigned, _Unit>(__parent),
|
||||
::cuda::experimental::__rank_query_group<unsigned, _Unit>(__parent),
|
||||
__parent.__mapping_result().lane_mask()};
|
||||
}
|
||||
|
||||
using _ParentMappingResult = typename _ParentGroup::__mapping_result_type;
|
||||
using _MappingResult = decltype(::cuda::std::declval<const _Mapping&>().map(
|
||||
::cuda::std::declval<const _Unit&>(),
|
||||
::cuda::std::declval<const _ParentGroup&>(),
|
||||
__get_initial_mapping_result(::cuda::std::declval<const _ParentGroup&>())));
|
||||
using _SynchronizerInstance =
|
||||
__group_synchronizer_instance_t<_Synchronizer, _Unit, _ParentGroup, _Mapping, _MappingResult>;
|
||||
static_assert(__group_mapping_result<_MappingResult>);
|
||||
|
||||
typename _ParentGroup::hierarchy_type __hier_;
|
||||
_Mapping __mapping_;
|
||||
_MappingResult __mapping_result_;
|
||||
_Synchronizer __synchronizer_;
|
||||
_SynchronizerInstance __synchronizer_instance_;
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static _MappingResult
|
||||
__do_mapping(const _Unit& __unit, const _Mapping& __mapping, const _ParentGroup& __parent) noexcept
|
||||
{
|
||||
const auto __mapping_result = __mapping.map(__unit, __parent, __get_initial_mapping_result(__parent));
|
||||
if (__mapping_result.is_valid())
|
||||
{
|
||||
_CCCL_ASSERT(__mapping_result.group_rank() < __mapping_result.group_count(), "invalid group rank");
|
||||
_CCCL_ASSERT(__mapping_result.unit_rank() < __mapping_result.unit_count(), "invalid unit rank");
|
||||
|
||||
if constexpr (::cuda::std::is_same_v<_Unit, thread_level>)
|
||||
{
|
||||
_CCCL_ASSERT(
|
||||
(__mapping_result.lane_mask() & ::cuda::device::lane_mask::this_lane()) != ::cuda::device::lane_mask::none(),
|
||||
"invalid lane mask - this lane must be contained in the lane mask");
|
||||
_CCCL_ASSERT(::cuda::std::popcount(__mapping_result.lane_mask().value()) <= __mapping_result.unit_count(),
|
||||
"invalid lane mask - too many lanes are set in the lane mask");
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_ASSERT(__mapping_result.lane_mask() == ::cuda::device::lane_mask::all(),
|
||||
"invalid lane mask - must be equal to cuda::device::lane_mask::all() when _Unit is not "
|
||||
"cuda::thread_level");
|
||||
}
|
||||
}
|
||||
return __mapping_result;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static _SynchronizerInstance __make_synchronizer_instance(
|
||||
const _Unit& __unit,
|
||||
const _Synchronizer& __synchronizer,
|
||||
const _ParentGroup& __parent,
|
||||
const _Mapping& __mapping,
|
||||
const _MappingResult& __mapping_result) noexcept
|
||||
{
|
||||
// Do not invoke the synchronizer instance creation for threads that are not part of the parent group. On the other
|
||||
// hand threads that are not part of this group must create the synchronizer instance, too, because the operation
|
||||
// can synchronize the parent group.
|
||||
if constexpr (!_ParentMappingResult::is_always_exhaustive())
|
||||
{
|
||||
if (!__parent.__mapping_result().is_valid())
|
||||
{
|
||||
return _SynchronizerInstance::invalid();
|
||||
}
|
||||
}
|
||||
return __synchronizer.make_instance(__unit, __parent, __mapping, __mapping_result);
|
||||
}
|
||||
|
||||
public:
|
||||
using unit_type = _Unit;
|
||||
using level_type = typename _ParentGroup::level_type;
|
||||
using hierarchy_type = typename _ParentGroup::hierarchy_type;
|
||||
using mapping_type = _Mapping;
|
||||
using __mapping_result_type = _MappingResult;
|
||||
using synchronizer_type = _Synchronizer;
|
||||
|
||||
_CCCL_DEVICE_API explicit group(
|
||||
const _Unit& __unit,
|
||||
const _ParentGroup& __parent,
|
||||
const _Mapping& __mapping,
|
||||
const _Synchronizer& __synchronizer) noexcept
|
||||
: __hier_{__parent.hierarchy()}
|
||||
, __mapping_{__mapping}
|
||||
, __mapping_result_{__do_mapping(__unit, __mapping_, __parent)}
|
||||
, __synchronizer_{__synchronizer}
|
||||
, __synchronizer_instance_{
|
||||
__make_synchronizer_instance(__unit, __synchronizer_, __parent, __mapping_, __mapping_result_)}
|
||||
{}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API const hierarchy_type& hierarchy() const noexcept
|
||||
{
|
||||
return __hier_;
|
||||
}
|
||||
|
||||
// todo(dabayer): Do we want to expose mapping getter?
|
||||
[[nodiscard]] _CCCL_DEVICE_API const mapping_type& mapping() const noexcept
|
||||
{
|
||||
return __mapping_;
|
||||
}
|
||||
|
||||
// todo(dabayer): Do we want to expose mapping result getter?
|
||||
[[nodiscard]] _CCCL_DEVICE_API _MappingResult __mapping_result() const noexcept
|
||||
{
|
||||
return __mapping_result_;
|
||||
}
|
||||
|
||||
// todo(dabayer): Do we want to expose synchronizer getter?
|
||||
[[nodiscard]] _CCCL_DEVICE_API const synchronizer_type& synchronizer() const noexcept
|
||||
{
|
||||
return __synchronizer_;
|
||||
}
|
||||
|
||||
// todo(dabayer): Do we want to expose .arrive() and .wait()? Do we want to implement .sync() using them? Do we want
|
||||
// aligned/unaligned variants?
|
||||
_CCCL_DEVICE_API void sync() const noexcept
|
||||
{
|
||||
// Skip the synchronization for threads that are not part of this group.
|
||||
if constexpr (!_MappingResult::is_always_exhaustive())
|
||||
{
|
||||
if (!__mapping_result_.is_valid())
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
__synchronizer_instance_.do_sync(__mapping_result_, __synchronizer_);
|
||||
}
|
||||
|
||||
_CCCL_DEVICE_API void sync_aligned() const noexcept
|
||||
{
|
||||
// Skip the synchronization for threads that are not part of this group.
|
||||
if constexpr (!_MappingResult::is_always_exhaustive())
|
||||
{
|
||||
if (!__mapping_result_.is_valid())
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
__synchronizer_instance_.do_sync_aligned(__mapping_result_, __synchronizer_);
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Tp, class _InLevel)
|
||||
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp> _CCCL_AND __is_hierarchy_level_v<_InLevel>)
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr _Tp count_as(const _InLevel&) const noexcept
|
||||
{
|
||||
_Tp __ret = __mapping_result_.group_count();
|
||||
if constexpr (!::cuda::std::is_same_v<_InLevel, level_type>)
|
||||
{
|
||||
__ret *= __count_query<level_type, _InLevel>::template __call<_Tp>(__hier_);
|
||||
}
|
||||
return __ret;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _InLevel)
|
||||
_CCCL_REQUIRES(__is_hierarchy_level_v<_InLevel>)
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr auto count(const _InLevel& __in_level) const noexcept
|
||||
{
|
||||
return count_as<typename _InLevel::__product_type>(__in_level);
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Tp, class _InLevel)
|
||||
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp> _CCCL_AND __is_hierarchy_level_v<_InLevel>)
|
||||
[[nodiscard]] _CCCL_DEVICE_API _Tp rank_as(const _InLevel&) const noexcept
|
||||
{
|
||||
_Tp __ret = __mapping_result_.group_rank();
|
||||
if constexpr (!::cuda::std::is_same_v<_InLevel, level_type>)
|
||||
{
|
||||
__ret += static_cast<_Tp>(
|
||||
__rank_query<level_type, _InLevel>::template __call<_Tp>(__hier_) * __mapping_result_.group_count());
|
||||
}
|
||||
return __ret;
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _InLevel)
|
||||
_CCCL_REQUIRES(__is_hierarchy_level_v<_InLevel>)
|
||||
[[nodiscard]] _CCCL_DEVICE_API auto rank(const _InLevel& __in_level) const noexcept
|
||||
{
|
||||
return rank_as<typename _InLevel::__product_type>(__in_level);
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_TEMPLATE(class _Unit, class _ParentGroup, class _Mapping, class _Synchronizer)
|
||||
_CCCL_REQUIRES(__is_hierarchy_level_v<_Unit> _CCCL_AND is_group<_ParentGroup>)
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES group(const _Unit&, const _ParentGroup&, const _Mapping&, const _Synchronizer&)
|
||||
-> group<_Unit, _ParentGroup, _Mapping, _Synchronizer>;
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_GROUP_CUH
|
||||
@@ -1,48 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_IMPLICIT_HIERARCHY_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_IMPLICIT_HIERARCHY_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/hierarchy>
|
||||
|
||||
#include <cuda/experimental/__group/fwd.cuh>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
[[nodiscard]] _CCCL_DEVICE_API inline __implicit_hierarchy_t __implicit_hierarchy() noexcept
|
||||
{
|
||||
return __implicit_hierarchy_t{
|
||||
gpu_thread,
|
||||
hierarchy_level_desc<grid_level, ::cuda::std::dims<3, unsigned>>{cluster.extents(grid)},
|
||||
hierarchy_level_desc<cluster_level, ::cuda::std::dims<3, unsigned>>{block.extents(cluster)},
|
||||
hierarchy_level_desc<block_level, ::cuda::std::dims<3, unsigned>>{gpu_thread.extents(block)}};
|
||||
}
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_IMPLICIT_HIERARCHY_CUH
|
||||
@@ -1,106 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_INVOKE_ONE_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_INVOKE_ONE_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/instructions/elect_sync.h>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__functional/invoke.h>
|
||||
#include <cuda/std/__type_traits/conditional.h>
|
||||
#include <cuda/std/__type_traits/is_reference.h>
|
||||
#include <cuda/std/__type_traits/is_same.h>
|
||||
#include <cuda/std/__type_traits/is_void.h>
|
||||
#include <cuda/std/__type_traits/remove_reference.h>
|
||||
#include <cuda/std/__utility/forward.h>
|
||||
#include <cuda/std/optional>
|
||||
|
||||
#include <cuda/experimental/__group/concepts.cuh>
|
||||
#include <cuda/experimental/__group/fwd.cuh>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
template <class _Group>
|
||||
[[nodiscard]] _CCCL_DEVICE_API bool __elect_one(const _Group& __group) noexcept
|
||||
{
|
||||
if constexpr (__is_this_group_v<_Group> && ::cuda::std::is_same_v<typename _Group::level_type, warp_level>)
|
||||
{
|
||||
NV_IF_TARGET(NV_PROVIDES_SM_90, ({ return ::cuda::ptx::elect_sync(~0u); }))
|
||||
}
|
||||
else if constexpr (!::cuda::std::is_same_v<typename _Group::unit_type, thread_level>)
|
||||
{
|
||||
// For groups whose unit is >= warp_level, we want to still execute the elect.sync PTX instruction by the root warp
|
||||
// to let the compiler enter the Uniform Data Path (UDP) when invoking the callable.
|
||||
NV_IF_TARGET(NV_PROVIDES_SM_90, ({
|
||||
if (warp.is_root_rank(__group))
|
||||
{
|
||||
return ::cuda::ptx::elect_sync(~0u);
|
||||
}
|
||||
return false;
|
||||
}))
|
||||
}
|
||||
return gpu_thread.is_root_rank(__group);
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Group, class _Callable, class... _Args)
|
||||
_CCCL_REQUIRES(is_group<_Group> _CCCL_AND ::cuda::std::is_invocable_v<_Callable, _Args...>
|
||||
_CCCL_AND ::cuda::std::is_void_v<::cuda::std::invoke_result_t<_Callable, _Args...>>)
|
||||
_CCCL_DEVICE_API void invoke_one(const _Group& __group, _Callable&& __callable, _Args&&... __args) noexcept(
|
||||
::cuda::std::is_nothrow_invocable_v<_Callable, _Args...>)
|
||||
{
|
||||
if (::cuda::experimental::__elect_one(__group))
|
||||
{
|
||||
::cuda::std::invoke(::cuda::std::forward<_Callable>(__callable), ::cuda::std::forward<_Args>(__args)...);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Group,
|
||||
class _Callable,
|
||||
class... _Args,
|
||||
class _InvokeResult = ::cuda::std::invoke_result_t<_Callable, _Args...>)
|
||||
_CCCL_REQUIRES(is_group<_Group> _CCCL_AND ::cuda::std::is_invocable_v<_Callable, _Args...> _CCCL_AND(
|
||||
!::cuda::std::is_void_v<_InvokeResult>))
|
||||
[[nodiscard]]
|
||||
_CCCL_DEVICE_API auto invoke_one(const _Group& __group, _Callable&& __callable, _Args&&... __args) noexcept(
|
||||
::cuda::std::is_nothrow_invocable_v<_Callable, _Args...>)
|
||||
{
|
||||
using _Ret = ::cuda::std::optional<::cuda::std::conditional_t<::cuda::std::is_rvalue_reference_v<_InvokeResult>,
|
||||
::cuda::std::remove_reference_t<_InvokeResult>,
|
||||
_InvokeResult>>;
|
||||
|
||||
_Ret __ret{};
|
||||
if (::cuda::experimental::__elect_one(__group))
|
||||
{
|
||||
__ret = ::cuda::std::invoke(::cuda::std::forward<_Callable>(__callable), ::cuda::std::forward<_Args>(__args)...);
|
||||
}
|
||||
return __ret;
|
||||
}
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_INVOKE_ONE_CUH
|
||||
@@ -1,103 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_MAPPING_BINARY_PARTITION_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_MAPPING_BINARY_PARTITION_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__ptx/instructions/get_sreg.h>
|
||||
#include <cuda/__warp/lane_mask.h>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/std/__bit/popcount.h>
|
||||
#include <cuda/std/__functional/invoke.h>
|
||||
#include <cuda/std/__fwd/span.h>
|
||||
#include <cuda/std/__type_traits/is_move_constructible.h>
|
||||
#include <cuda/std/__type_traits/is_same.h>
|
||||
#include <cuda/std/__utility/move.h>
|
||||
|
||||
#include <cuda/experimental/__group/fwd.cuh>
|
||||
#include <cuda/experimental/__group/mapping/mapping_result.cuh>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
template <class _Fn>
|
||||
class binary_partition
|
||||
{
|
||||
static_assert(::cuda::std::is_move_constructible_v<_Fn>, "_Fn must be move constructible");
|
||||
|
||||
mutable _Fn __fn_;
|
||||
|
||||
public:
|
||||
_CCCL_DEVICE_API explicit binary_partition(_Fn __fn) noexcept(::cuda::std::is_nothrow_move_constructible_v<_Fn>)
|
||||
: __fn_(::cuda::std::move(__fn))
|
||||
{}
|
||||
|
||||
template <class _Unit, class _ParentGroup, class _PrevMappingResult>
|
||||
[[nodiscard]] _CCCL_DEVICE_API auto
|
||||
map(const _Unit&, const _ParentGroup& __parent, const _PrevMappingResult& __prev_mapping_result) const
|
||||
noexcept(::cuda::std::is_nothrow_invocable_v<_Fn, const _PrevMappingResult&>)
|
||||
{
|
||||
static_assert(::cuda::std::is_same_v<_Unit, thread_level>, "binary_partition can only group threads");
|
||||
static_assert(::cuda::std::is_same_v<typename _ParentGroup::level_type, warp_level>,
|
||||
"binary_partition can be only used within warp_level");
|
||||
|
||||
constexpr auto __static_prev_ngroups = _PrevMappingResult::static_group_count();
|
||||
constexpr auto __static_ngroups =
|
||||
(__static_prev_ngroups != ::cuda::std::dynamic_extent)
|
||||
? (__static_prev_ngroups * 2)
|
||||
: ::cuda::std::dynamic_extent;
|
||||
|
||||
using _MappingResult =
|
||||
__mapping_result<__static_ngroups, ::cuda::std::dynamic_extent, _PrevMappingResult::is_always_exhaustive(), false>;
|
||||
|
||||
if (!__prev_mapping_result.is_valid())
|
||||
{
|
||||
return _MappingResult::invalid();
|
||||
}
|
||||
|
||||
const auto __pred = static_cast<bool>(__fn_(__prev_mapping_result));
|
||||
const auto __prev_mask = __prev_mapping_result.lane_mask().value();
|
||||
|
||||
auto __match_mask = ::__ballot_sync(__prev_mask, __pred);
|
||||
if (!__pred)
|
||||
{
|
||||
__match_mask = (~__match_mask) & __prev_mask;
|
||||
}
|
||||
return _MappingResult{
|
||||
__prev_mapping_result.group_count() * 2,
|
||||
__prev_mapping_result.group_rank() + ((__pred) ? __prev_mapping_result.group_count() : 0u),
|
||||
static_cast<unsigned>(::cuda::std::popcount(__match_mask)),
|
||||
static_cast<unsigned>(::cuda::std::popcount(__match_mask & ::cuda::ptx::get_sreg_lanemask_lt())),
|
||||
::cuda::device::lane_mask{__match_mask}};
|
||||
}
|
||||
};
|
||||
|
||||
template <class _PredFn>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES binary_partition(_PredFn) -> binary_partition<_PredFn>;
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_MAPPING_BINARY_PARTITION_CUH
|
||||
@@ -1,140 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_MAPPING_COMPOSITE_MAPPING_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_MAPPING_COMPOSITE_MAPPING_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__type_traits/fold.h>
|
||||
#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
|
||||
#include <cuda/std/tuple>
|
||||
|
||||
#include <cuda/experimental/__group/fwd.cuh>
|
||||
#include <cuda/experimental/__group/queries.cuh>
|
||||
#include <cuda/experimental/__group/traits.cuh>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
// todo(dabayer): do we want to always use uint32_t for all counts/ranks?
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
template <class... _Mappings>
|
||||
class composite_mapping
|
||||
{
|
||||
::cuda::std::tuple<_Mappings...> __mappings_;
|
||||
|
||||
template <::cuda::std::size_t _Ip = 0, class _Unit, class _ParentGroup, class _PrevMappingResult>
|
||||
[[nodiscard]] _CCCL_DEVICE_API auto __map_impl(
|
||||
const _Unit& __unit, const _ParentGroup& __parent, const _PrevMappingResult& __prev_mapping_result) const noexcept
|
||||
{
|
||||
const auto __result = ::cuda::std::get<_Ip>(__mappings_).map(__unit, __parent, __prev_mapping_result);
|
||||
if constexpr (_Ip + 1 < sizeof...(_Mappings))
|
||||
{
|
||||
return __map_impl<_Ip + 1>(__unit, __parent, __result);
|
||||
}
|
||||
else
|
||||
{
|
||||
return __result;
|
||||
}
|
||||
}
|
||||
|
||||
public:
|
||||
_CCCL_DEVICE_API constexpr composite_mapping(const _Mappings&... __mappings) noexcept(
|
||||
::cuda::std::__fold_and_v<::cuda::std::is_nothrow_copy_constructible_v<_Mappings>...>)
|
||||
: __mappings_{__mappings...}
|
||||
{}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr const ::cuda::std::tuple<_Mappings...>& get() const noexcept
|
||||
{
|
||||
return __mappings_;
|
||||
}
|
||||
|
||||
template <class _Unit, class _ParentGroup, class _PrevMappingResult>
|
||||
[[nodiscard]] _CCCL_DEVICE_API auto
|
||||
map(const _Unit& __unit, const _ParentGroup& __parent, const _PrevMappingResult& __prev_mapping_result) const noexcept
|
||||
{
|
||||
return __map_impl(__unit, __parent, __prev_mapping_result);
|
||||
}
|
||||
};
|
||||
|
||||
template <class... _Mappings>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES composite_mapping(const _Mappings&...) -> composite_mapping<_Mappings...>;
|
||||
|
||||
_CCCL_TEMPLATE(class _Lhs, class _Rhs)
|
||||
_CCCL_REQUIRES(__is_group_mapping_v<_Lhs> _CCCL_AND __is_group_mapping_v<_Rhs>)
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr composite_mapping<_Lhs, _Rhs>
|
||||
operator|(const _Lhs& __lhs, const _Rhs& __rhs) noexcept(
|
||||
::cuda::std::is_nothrow_constructible_v<composite_mapping<_Lhs, _Rhs>, const _Lhs&, const _Rhs&>)
|
||||
{
|
||||
return {__lhs, __rhs};
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class... _LhsMappings, class _Rhs)
|
||||
_CCCL_REQUIRES(__is_group_mapping_v<_Rhs>)
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr composite_mapping<_LhsMappings..., _Rhs>
|
||||
operator|(const composite_mapping<_LhsMappings...>& __lhs, const _Rhs& __rhs) noexcept(
|
||||
::cuda::std::is_nothrow_constructible_v<composite_mapping<_LhsMappings..., _Rhs>, const _LhsMappings&..., const _Rhs&>)
|
||||
{
|
||||
return ::cuda::std::apply(
|
||||
[&](const auto&... __lhs_mappings) {
|
||||
return composite_mapping{__lhs_mappings..., __rhs};
|
||||
},
|
||||
__lhs.get());
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _Lhs, class... _RhsMappings)
|
||||
_CCCL_REQUIRES(__is_group_mapping_v<_Lhs>)
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr composite_mapping<_Lhs, _RhsMappings...>
|
||||
operator|(const _Lhs& __lhs, const composite_mapping<_RhsMappings...>& __rhs) noexcept(
|
||||
::cuda::std::is_nothrow_constructible_v<composite_mapping<_Lhs, _RhsMappings...>, const _Lhs&, const _RhsMappings&...>)
|
||||
{
|
||||
return ::cuda::std::apply(
|
||||
[&](const auto&... __rhs_mappings) {
|
||||
return composite_mapping{__lhs, __rhs_mappings...};
|
||||
},
|
||||
__rhs.get());
|
||||
}
|
||||
|
||||
template <class... _LhsMappings, class... _RhsMappings>
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr composite_mapping<_LhsMappings..., _RhsMappings...>
|
||||
operator|(const composite_mapping<_LhsMappings...>& __lhs, const composite_mapping<_RhsMappings...>& __rhs) noexcept(
|
||||
::cuda::std::is_nothrow_constructible_v<composite_mapping<_LhsMappings..., _RhsMappings...>,
|
||||
const _LhsMappings&...,
|
||||
const _RhsMappings&...>)
|
||||
{
|
||||
return ::cuda::std::apply(
|
||||
[&](const auto&... __lhs_mappings) {
|
||||
return ::cuda::std::apply(
|
||||
[&](const auto&... __rhs_mappings) {
|
||||
return composite_mapping{__lhs_mappings..., __rhs_mappings...};
|
||||
},
|
||||
__rhs.get());
|
||||
},
|
||||
__lhs.get());
|
||||
}
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_MAPPING_COMPOSITE_MAPPING_CUH
|
||||
@@ -1,330 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_MAPPING_GROUP_AS_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_MAPPING_GROUP_AS_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/std/__cstddef/types.h>
|
||||
#include <cuda/std/__exception/exception_macros.h>
|
||||
#include <cuda/std/__host_stdlib/stdexcept>
|
||||
#include <cuda/std/__numeric/accumulate.h>
|
||||
#include <cuda/std/__utility/cmp.h>
|
||||
#include <cuda/std/__utility/integer_sequence.h>
|
||||
#include <cuda/std/span>
|
||||
|
||||
#include <cuda/experimental/__group/fwd.cuh>
|
||||
#include <cuda/experimental/__group/mapping/mapping_result.cuh>
|
||||
#include <cuda/experimental/__group/queries.cuh>
|
||||
#include <cuda/experimental/__group/traits.cuh>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
// todo(dabayer): do we want to always use uint32_t for all counts/ranks?
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
template <::cuda::std::size_t... _UnitCounts>
|
||||
struct __group_as_static_tag;
|
||||
|
||||
template <::cuda::std::size_t... _UnitCounts, bool _IsExhaustive>
|
||||
class group_as<__group_as_static_tag<_UnitCounts...>, _IsExhaustive>
|
||||
{
|
||||
static_assert(((_UnitCounts != 0) && ...), "all _UnitCounts must not be zero");
|
||||
static_assert((::cuda::std::in_range<unsigned>(_UnitCounts) && ...), "all _UnitCounts must be within uint32_t range");
|
||||
|
||||
static constexpr auto __counts_sum = (0 + ... + _UnitCounts);
|
||||
|
||||
public:
|
||||
_CCCL_HIDE_FROM_ABI explicit group_as() = default;
|
||||
|
||||
_CCCL_TEMPLATE(bool _IsExhaustive2 = _IsExhaustive)
|
||||
_CCCL_REQUIRES(_IsExhaustive2)
|
||||
_CCCL_DEVICE_API explicit constexpr group_as(
|
||||
const ::cuda::std::integer_sequence<::cuda::std::size_t, _UnitCounts...>&) noexcept
|
||||
{}
|
||||
|
||||
_CCCL_TEMPLATE(bool _IsExhaustive2 = _IsExhaustive)
|
||||
_CCCL_REQUIRES((!_IsExhaustive2))
|
||||
_CCCL_DEVICE_API explicit constexpr group_as(
|
||||
const ::cuda::std::integer_sequence<::cuda::std::size_t, _UnitCounts...>&, const non_exhaustive_t&) noexcept
|
||||
{}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr ::cuda::std::size_t static_group_count() noexcept
|
||||
{
|
||||
return sizeof...(_UnitCounts);
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr ::cuda::std::size_t static_unit_count(::cuda::std::size_t __i) noexcept
|
||||
{
|
||||
if (__i >= sizeof...(_UnitCounts))
|
||||
{
|
||||
_CCCL_THROW(::std::out_of_range, "__i is out of range");
|
||||
}
|
||||
constexpr ::cuda::std::size_t __counts[]{_UnitCounts...};
|
||||
return __counts[__i];
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr bool is_always_exhaustive() noexcept
|
||||
{
|
||||
return _IsExhaustive;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr unsigned unit_count(::cuda::std::size_t __i) const noexcept
|
||||
{
|
||||
return static_cast<unsigned>(static_unit_count(__i));
|
||||
}
|
||||
|
||||
template <class _Unit, class _ParentGroup, class _PrevMappingResult>
|
||||
[[nodiscard]] _CCCL_DEVICE_API auto
|
||||
map(const _Unit&, const _ParentGroup&, const _PrevMappingResult& __prev_mapping_result) const noexcept
|
||||
{
|
||||
constexpr auto __static_prev_ngroups = _PrevMappingResult::static_group_count();
|
||||
constexpr auto __static_prev_nunits = _PrevMappingResult::static_unit_count();
|
||||
constexpr auto __static_curr_ngroups = sizeof...(_UnitCounts);
|
||||
constexpr auto __static_ngroups =
|
||||
(__static_prev_ngroups != ::cuda::std::dynamic_extent)
|
||||
? (__static_prev_ngroups * __static_curr_ngroups)
|
||||
: ::cuda::std::dynamic_extent;
|
||||
|
||||
using _MappingResult =
|
||||
__mapping_result<__static_ngroups,
|
||||
::cuda::std::dynamic_extent,
|
||||
_PrevMappingResult::is_always_exhaustive() && _IsExhaustive,
|
||||
_PrevMappingResult::is_always_contiguous()>;
|
||||
|
||||
if (!__prev_mapping_result.is_valid())
|
||||
{
|
||||
return _MappingResult::invalid();
|
||||
}
|
||||
|
||||
const auto __prev_nunits = __prev_mapping_result.unit_count();
|
||||
const auto __prev_unit_rank = __prev_mapping_result.unit_rank();
|
||||
constexpr auto __curr_ngroups = static_cast<unsigned>(sizeof...(_UnitCounts));
|
||||
const auto __ngroups = __prev_mapping_result.group_count() * __curr_ngroups;
|
||||
|
||||
if constexpr (_IsExhaustive)
|
||||
{
|
||||
if constexpr (__static_prev_nunits != ::cuda::std::dynamic_extent)
|
||||
{
|
||||
static_assert(__static_prev_nunits == __counts_sum, "group_as mapping _IsExhaustive precondition violation");
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_ASSERT(__prev_nunits == static_cast<unsigned>(__counts_sum),
|
||||
"group_as mapping _IsExhaustive precondition violation");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (__static_prev_nunits != ::cuda::std::dynamic_extent)
|
||||
{
|
||||
static_assert(__static_prev_nunits >= __counts_sum, "group_as mapping requires more units than are available");
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_ASSERT(__prev_nunits >= static_cast<unsigned>(__counts_sum),
|
||||
"group_as mapping requires more units than are available");
|
||||
}
|
||||
|
||||
if (__prev_unit_rank >= static_cast<unsigned>(__counts_sum))
|
||||
{
|
||||
return _MappingResult::invalid_with_group_count(__ngroups);
|
||||
}
|
||||
}
|
||||
|
||||
unsigned __sum = 0;
|
||||
_CCCL_PRAGMA_UNROLL_FULL()
|
||||
for (unsigned __i = 0; __i < __curr_ngroups; ++__i)
|
||||
{
|
||||
const auto __i_count = unit_count(__i);
|
||||
if (__prev_unit_rank < __sum + __i_count)
|
||||
{
|
||||
const auto __group_rank = __prev_mapping_result.group_rank() * __curr_ngroups + __i;
|
||||
const auto __n = __i_count;
|
||||
const auto __rank = __prev_unit_rank - __sum;
|
||||
const auto __lane_mask =
|
||||
(::cuda::std::is_same_v<_Unit, thread_level>)
|
||||
? ::cuda::experimental::__make_lane_mask_for_n<_PrevMappingResult::is_always_contiguous()>(
|
||||
__prev_mapping_result.lane_mask(), __n, __rank)
|
||||
: __prev_mapping_result.lane_mask();
|
||||
return _MappingResult{__ngroups, __group_rank, __n, __rank, __lane_mask};
|
||||
}
|
||||
__sum += __i_count;
|
||||
}
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
};
|
||||
|
||||
template <::cuda::std::size_t _GroupCount>
|
||||
struct __group_as_dynamic_tag;
|
||||
|
||||
template <::cuda::std::size_t _GroupCount, bool _IsExhaustive>
|
||||
class group_as<__group_as_dynamic_tag<_GroupCount>, _IsExhaustive>
|
||||
{
|
||||
static_assert(_GroupCount != ::cuda::std::dynamic_extent, "group_as requires static number of groups");
|
||||
|
||||
unsigned __counts_[_GroupCount];
|
||||
|
||||
public:
|
||||
_CCCL_TEMPLATE(bool _IsExhaustive2 = _IsExhaustive)
|
||||
_CCCL_REQUIRES(_IsExhaustive2)
|
||||
_CCCL_DEVICE_API explicit constexpr group_as(::cuda::std::span<const unsigned, _GroupCount> __counts) noexcept
|
||||
{
|
||||
_CCCL_PRAGMA_UNROLL_FULL()
|
||||
for (::cuda::std::size_t __i = 0; __i < _GroupCount; ++__i)
|
||||
{
|
||||
_CCCL_ASSERT(__counts[__i] > 0, "none of the __counts can be 0");
|
||||
__counts_[__i] = __counts[__i];
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(bool _IsExhaustive2 = _IsExhaustive)
|
||||
_CCCL_REQUIRES((!_IsExhaustive2))
|
||||
_CCCL_DEVICE_API explicit constexpr group_as(::cuda::std::span<const unsigned, _GroupCount> __counts,
|
||||
const non_exhaustive_t&) noexcept
|
||||
{
|
||||
_CCCL_PRAGMA_UNROLL_FULL()
|
||||
for (::cuda::std::size_t __i = 0; __i < _GroupCount; ++__i)
|
||||
{
|
||||
_CCCL_ASSERT(__counts[__i] > 0, "none of the __counts can be 0");
|
||||
__counts_[__i] = __counts[__i];
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr ::cuda::std::size_t static_group_count() noexcept
|
||||
{
|
||||
return _GroupCount;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr ::cuda::std::size_t static_unit_count(::cuda::std::size_t __i) noexcept
|
||||
{
|
||||
if (__i >= _GroupCount)
|
||||
{
|
||||
_CCCL_THROW(::std::out_of_range, "__i is out of range");
|
||||
}
|
||||
return ::cuda::std::dynamic_extent;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr bool is_always_exhaustive() noexcept
|
||||
{
|
||||
return _IsExhaustive;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr unsigned unit_count(::cuda::std::size_t __i) const noexcept
|
||||
{
|
||||
if (__i >= _GroupCount)
|
||||
{
|
||||
_CCCL_THROW(::std::out_of_range, "__i is out of range");
|
||||
}
|
||||
return __counts_[__i];
|
||||
}
|
||||
|
||||
template <class _Unit, class _ParentGroup, class _PrevMappingResult>
|
||||
[[nodiscard]] _CCCL_DEVICE_API auto
|
||||
map(const _Unit&, const _ParentGroup&, const _PrevMappingResult& __prev_mapping_result) const noexcept
|
||||
{
|
||||
constexpr auto __static_prev_ngroups = _PrevMappingResult::static_group_count();
|
||||
constexpr auto __static_prev_nunits = _PrevMappingResult::static_unit_count();
|
||||
constexpr auto __static_curr_ngroups = _GroupCount;
|
||||
constexpr auto __static_ngroups =
|
||||
(__static_prev_ngroups != ::cuda::std::dynamic_extent)
|
||||
? (__static_prev_ngroups * __static_curr_ngroups)
|
||||
: ::cuda::std::dynamic_extent;
|
||||
|
||||
using _MappingResult =
|
||||
__mapping_result<__static_ngroups,
|
||||
::cuda::std::dynamic_extent,
|
||||
_PrevMappingResult::is_always_exhaustive() && _IsExhaustive,
|
||||
_PrevMappingResult::is_always_contiguous()>;
|
||||
|
||||
if (!__prev_mapping_result.is_valid())
|
||||
{
|
||||
return _MappingResult::invalid();
|
||||
}
|
||||
|
||||
const auto __prev_nunits = __prev_mapping_result.unit_count();
|
||||
const auto __prev_unit_rank = __prev_mapping_result.unit_rank();
|
||||
constexpr auto __curr_ngroups = static_cast<unsigned>(_GroupCount);
|
||||
const auto __ngroups = __prev_mapping_result.group_count() * __curr_ngroups;
|
||||
|
||||
// If the mapping is exhaustive, check the preconditions, otherwise remove the last partial group.
|
||||
if constexpr (_IsExhaustive)
|
||||
{
|
||||
_CCCL_ASSERT(::cuda::std::accumulate(__counts_, __counts_ + __curr_ngroups, 0u) == __prev_nunits,
|
||||
"group_as mapping _IsExhaustive precondition violation");
|
||||
}
|
||||
else if (__prev_unit_rank >= ::cuda::std::accumulate(__counts_, __counts_ + __curr_ngroups, 0u))
|
||||
{
|
||||
return _MappingResult::invalid_with_group_count(__ngroups);
|
||||
}
|
||||
|
||||
unsigned __sum = 0;
|
||||
_CCCL_PRAGMA_UNROLL_FULL()
|
||||
for (unsigned __i = 0; __i < __curr_ngroups; ++__i)
|
||||
{
|
||||
const auto __i_count = unit_count(__i);
|
||||
if (__prev_unit_rank < __sum + __i_count)
|
||||
{
|
||||
const auto __group_rank = __prev_mapping_result.group_rank() * __curr_ngroups + __i;
|
||||
const auto __n = __i_count;
|
||||
const auto __rank = __prev_unit_rank - __sum;
|
||||
const auto __lane_mask =
|
||||
(::cuda::std::is_same_v<_Unit, thread_level>)
|
||||
? ::cuda::experimental::__make_lane_mask_for_n<_PrevMappingResult::is_always_contiguous()>(
|
||||
__prev_mapping_result.lane_mask(), __n, __rank)
|
||||
: __prev_mapping_result.lane_mask();
|
||||
return _MappingResult{__ngroups, __group_rank, __n, __rank, __lane_mask};
|
||||
}
|
||||
__sum += __i_count;
|
||||
}
|
||||
_CCCL_UNREACHABLE();
|
||||
}
|
||||
};
|
||||
|
||||
template <::cuda::std::size_t... _UnitCounts>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES group_as(const ::cuda::std::integer_sequence<::cuda::std::size_t, _UnitCounts...>&)
|
||||
-> group_as<__group_as_static_tag<_UnitCounts...>, true>;
|
||||
|
||||
template <::cuda::std::size_t... _UnitCounts>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES
|
||||
group_as(const ::cuda::std::integer_sequence<::cuda::std::size_t, _UnitCounts...>&, const non_exhaustive_t&)
|
||||
-> group_as<__group_as_static_tag<_UnitCounts...>, false>;
|
||||
|
||||
_CCCL_TEMPLATE(class _Tp)
|
||||
_CCCL_REQUIRES(__is_spannable<_Tp> _CCCL_AND ::cuda::std::
|
||||
is_same_v<unsigned, _SpanValueType<decltype(::cuda::std::span(::cuda::std::declval<_Tp&>()))>>)
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES group_as(_Tp& __v)
|
||||
-> group_as<__group_as_dynamic_tag<decltype(::cuda::std::span(__v))::extent>, true>;
|
||||
|
||||
_CCCL_TEMPLATE(class _Tp)
|
||||
_CCCL_REQUIRES(__is_spannable<_Tp> _CCCL_AND ::cuda::std::
|
||||
is_same_v<unsigned, _SpanValueType<decltype(::cuda::std::span(::cuda::std::declval<_Tp&>()))>>)
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES group_as(_Tp& __v, const non_exhaustive_t&)
|
||||
-> group_as<__group_as_dynamic_tag<decltype(::cuda::std::span(__v))::extent>, false>;
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_MAPPING_GROUP_AS_CUH
|
||||
@@ -1,237 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_MAPPING_GROUP_BY_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_MAPPING_GROUP_BY_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/std/__cstddef/types.h>
|
||||
#include <cuda/std/__fwd/span.h>
|
||||
#include <cuda/std/__utility/cmp.h>
|
||||
|
||||
#include <cuda/experimental/__group/fwd.cuh>
|
||||
#include <cuda/experimental/__group/mapping/mapping_result.cuh>
|
||||
#include <cuda/experimental/__group/queries.cuh>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
// todo(dabayer): do we want to always use uint32_t for all counts/ranks?
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
struct non_exhaustive_t
|
||||
{
|
||||
_CCCL_HIDE_FROM_ABI explicit non_exhaustive_t() = default;
|
||||
};
|
||||
|
||||
_CCCL_DEVICE constexpr non_exhaustive_t non_exhaustive;
|
||||
|
||||
// Requirements on mappings:
|
||||
// - must be copyable
|
||||
// - must implement `map(_Unit, _Level, _Hierarchy)` method that returns an object that satisfies the
|
||||
// `__group_mapping_result` concept
|
||||
|
||||
// todo(dabayer): do we want to add stride parameter?
|
||||
template <::cuda::std::size_t _UnitCount, bool _IsExhaustive>
|
||||
class group_by
|
||||
{
|
||||
static_assert(_UnitCount != 0, "_UnitCount must not be zero");
|
||||
static_assert(::cuda::std::in_range<unsigned>(_UnitCount), "_UnitCount must be within uint32_t range");
|
||||
|
||||
public:
|
||||
_CCCL_HIDE_FROM_ABI explicit group_by() = default;
|
||||
|
||||
_CCCL_TEMPLATE(bool _IsExhaustive2 = _IsExhaustive)
|
||||
_CCCL_REQUIRES((!_IsExhaustive))
|
||||
_CCCL_DEVICE_API constexpr group_by(const non_exhaustive_t&) noexcept {}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr ::cuda::std::size_t static_unit_count() noexcept
|
||||
{
|
||||
return _UnitCount;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr bool is_always_exhaustive() noexcept
|
||||
{
|
||||
return _IsExhaustive;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr unsigned unit_count() const noexcept
|
||||
{
|
||||
return static_cast<unsigned>(_UnitCount);
|
||||
}
|
||||
|
||||
template <class _Unit, class _ParentGroup, class _PrevMappingResult>
|
||||
[[nodiscard]] _CCCL_DEVICE_API auto
|
||||
map(const _Unit&, const _ParentGroup& __parent, const _PrevMappingResult& __prev_mapping_result) const noexcept
|
||||
{
|
||||
constexpr auto __static_prev_ngroups = _PrevMappingResult::static_group_count();
|
||||
constexpr auto __static_prev_nunits = _PrevMappingResult::static_unit_count();
|
||||
constexpr auto __static_curr_ngroups =
|
||||
(__static_prev_nunits != ::cuda::std::dynamic_extent)
|
||||
? __static_prev_nunits / _UnitCount
|
||||
: ::cuda::std::dynamic_extent;
|
||||
constexpr auto __static_ngroups =
|
||||
(__static_prev_ngroups != ::cuda::std::dynamic_extent && __static_curr_ngroups != ::cuda::std::dynamic_extent)
|
||||
? (__static_prev_ngroups * __static_curr_ngroups)
|
||||
: ::cuda::std::dynamic_extent;
|
||||
|
||||
using _MappingResult =
|
||||
__mapping_result<__static_ngroups,
|
||||
_UnitCount,
|
||||
_PrevMappingResult::is_always_exhaustive() && _IsExhaustive,
|
||||
_PrevMappingResult::is_always_contiguous()>;
|
||||
|
||||
if (!__prev_mapping_result.is_valid())
|
||||
{
|
||||
return _MappingResult::invalid();
|
||||
}
|
||||
|
||||
const auto __prev_nunits = __prev_mapping_result.unit_count();
|
||||
const auto __prev_unit_rank = __prev_mapping_result.unit_rank();
|
||||
const auto __curr_ngroups = __prev_nunits / unit_count();
|
||||
const auto __curr_group_rank = __prev_unit_rank / unit_count();
|
||||
const auto __ngroups = __prev_mapping_result.group_count() * __curr_ngroups;
|
||||
|
||||
// If the mapping is exhaustive, check the preconditions, otherwise return invalid mapping for the remainder.
|
||||
if constexpr (_IsExhaustive)
|
||||
{
|
||||
if constexpr (__static_prev_nunits != ::cuda::std::dynamic_extent)
|
||||
{
|
||||
static_assert(__static_prev_nunits % _UnitCount == 0, "group_by mapping _IsExhaustive precondition violation");
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_ASSERT(__prev_nunits % unit_count() == 0, "group_by mapping _IsExhaustive precondition violation");
|
||||
}
|
||||
}
|
||||
else if (__prev_nunits % unit_count() != 0)
|
||||
{
|
||||
if (__curr_group_rank >= __curr_ngroups)
|
||||
{
|
||||
return _MappingResult::invalid_with_group_count(__ngroups);
|
||||
}
|
||||
}
|
||||
|
||||
const auto __group_rank = __prev_mapping_result.group_rank() * __curr_ngroups + __curr_group_rank;
|
||||
const auto __n = unit_count();
|
||||
const auto __rank = __prev_unit_rank % __n;
|
||||
const auto __lane_mask =
|
||||
(::cuda::std::is_same_v<_Unit, thread_level>)
|
||||
? ::cuda::experimental::__make_lane_mask_for_n<_PrevMappingResult::is_always_contiguous()>(
|
||||
__prev_mapping_result.lane_mask(), __n, __rank)
|
||||
: __prev_mapping_result.lane_mask();
|
||||
return _MappingResult{__ngroups, __group_rank, __n, __rank, __lane_mask};
|
||||
}
|
||||
};
|
||||
|
||||
template <bool _IsExhaustive>
|
||||
class group_by<::cuda::std::dynamic_extent, _IsExhaustive>
|
||||
{
|
||||
unsigned __count_;
|
||||
|
||||
public:
|
||||
_CCCL_DEVICE_API explicit constexpr group_by(unsigned __count) noexcept
|
||||
: __count_{__count}
|
||||
{
|
||||
_CCCL_ASSERT(__count > 0, "__count cannot be 0");
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(bool _IsExhaustive2 = _IsExhaustive)
|
||||
_CCCL_REQUIRES((!_IsExhaustive2))
|
||||
_CCCL_DEVICE_API explicit constexpr group_by(unsigned __count, const non_exhaustive_t&) noexcept
|
||||
: __count_{__count}
|
||||
{
|
||||
_CCCL_ASSERT(__count > 0, "__count cannot be 0");
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr ::cuda::std::size_t static_unit_count() noexcept
|
||||
{
|
||||
return ::cuda::std::dynamic_extent;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr bool is_always_exhaustive() noexcept
|
||||
{
|
||||
return _IsExhaustive;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr unsigned unit_count() const noexcept
|
||||
{
|
||||
return __count_;
|
||||
}
|
||||
|
||||
template <class _Unit, class _ParentGroup, class _PrevMappingResult>
|
||||
[[nodiscard]] _CCCL_DEVICE_API auto
|
||||
map(const _Unit&, const _ParentGroup& __parent, const _PrevMappingResult& __prev_mapping_result) const noexcept
|
||||
{
|
||||
using _MappingResult =
|
||||
__mapping_result<::cuda::std::dynamic_extent,
|
||||
::cuda::std::dynamic_extent,
|
||||
_PrevMappingResult::is_always_exhaustive() && _IsExhaustive,
|
||||
_PrevMappingResult::is_always_contiguous()>;
|
||||
|
||||
if (!__prev_mapping_result.is_valid())
|
||||
{
|
||||
return _MappingResult::invalid();
|
||||
}
|
||||
|
||||
const auto __prev_nunits = __prev_mapping_result.unit_count();
|
||||
const auto __prev_unit_rank = __prev_mapping_result.unit_rank();
|
||||
const auto __curr_ngroups = __prev_nunits / __count_;
|
||||
const auto __curr_group_rank = __prev_unit_rank / __count_;
|
||||
const auto __ngroups = __prev_mapping_result.group_count() * __curr_ngroups;
|
||||
|
||||
// If the mapping is exhaustive, check the preconditions, otherwise remove the last partial group.
|
||||
if constexpr (_IsExhaustive)
|
||||
{
|
||||
_CCCL_ASSERT(__prev_nunits % __count_ == 0, "group_by mapping _IsExhaustive precondition violation");
|
||||
}
|
||||
else if (__prev_nunits % __count_ != 0)
|
||||
{
|
||||
if (__curr_group_rank >= __curr_ngroups)
|
||||
{
|
||||
return _MappingResult::invalid_with_group_count(__ngroups);
|
||||
}
|
||||
}
|
||||
|
||||
const auto __group_rank = __prev_mapping_result.group_rank() * __curr_ngroups + __curr_group_rank;
|
||||
const auto __n = __count_;
|
||||
const auto __rank = __prev_unit_rank % __count_;
|
||||
const auto __lane_mask =
|
||||
(::cuda::std::is_same_v<_Unit, thread_level>)
|
||||
? ::cuda::experimental::__make_lane_mask_for_n<_PrevMappingResult::is_always_contiguous()>(
|
||||
__prev_mapping_result.lane_mask(), __n, __rank)
|
||||
: __prev_mapping_result.lane_mask();
|
||||
return _MappingResult{__ngroups, __group_rank, __n, __rank, __lane_mask};
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES group_by(unsigned) -> group_by<::cuda::std::dynamic_extent>;
|
||||
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES group_by(unsigned, const non_exhaustive_t&)
|
||||
-> group_by<::cuda::std::dynamic_extent, false>;
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_MAPPING_GROUP_BY_CUH
|
||||
@@ -1,50 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_MAPPING_IDENTITY_MAPPING_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_MAPPING_IDENTITY_MAPPING_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/experimental/__group/fwd.cuh>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
class identity_mapping
|
||||
{
|
||||
public:
|
||||
_CCCL_HIDE_FROM_ABI explicit identity_mapping() = default;
|
||||
|
||||
template <class _Unit, class _ParentGroup, class _PrevMappingResult>
|
||||
[[nodiscard]] _CCCL_DEVICE_API auto
|
||||
map(const _Unit&, const _ParentGroup&, const _PrevMappingResult& __prev_mapping_result) const noexcept
|
||||
{
|
||||
return __prev_mapping_result;
|
||||
}
|
||||
};
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_MAPPING_IDENTITY_MAPPING_CUH
|
||||
@@ -1,217 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_MAPPING_MAPPING_RESULT_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_MAPPING_MAPPING_RESULT_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__warp/lane_mask.h>
|
||||
#include <cuda/std/__bit/popcount.h>
|
||||
#include <cuda/std/__cstddef/types.h>
|
||||
#include <cuda/std/__fwd/span.h>
|
||||
|
||||
#include <cuda/experimental/__group/fwd.cuh>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
// todo(dabayer): do we want to always use uint32_t for all counts/ranks?
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
template <::cuda::std::size_t _StaticGroupCount, ::cuda::std::size_t _StaticCount, bool _IsExhaustive, bool _IsContiguous>
|
||||
struct __mapping_result
|
||||
{
|
||||
unsigned __group_count_;
|
||||
unsigned __group_rank_;
|
||||
unsigned __unit_count_;
|
||||
unsigned __unit_rank_;
|
||||
::cuda::device::lane_mask __lane_mask_;
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr __mapping_result invalid() noexcept
|
||||
{
|
||||
return {__invalid_count_or_rank,
|
||||
__invalid_count_or_rank,
|
||||
__invalid_count_or_rank,
|
||||
__invalid_count_or_rank,
|
||||
::cuda::device::lane_mask::none()};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr __mapping_result
|
||||
invalid_with_group_count(unsigned __group_count) noexcept
|
||||
{
|
||||
return {__group_count,
|
||||
__invalid_count_or_rank,
|
||||
__invalid_count_or_rank,
|
||||
__invalid_count_or_rank,
|
||||
::cuda::device::lane_mask::none()};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr ::cuda::std::size_t static_group_count() noexcept
|
||||
{
|
||||
return _StaticGroupCount;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API unsigned group_count() const noexcept
|
||||
{
|
||||
if constexpr (_StaticGroupCount != ::cuda::std::dynamic_extent)
|
||||
{
|
||||
return static_cast<unsigned>(_StaticGroupCount);
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (!_IsExhaustive)
|
||||
{
|
||||
_CCCL_ASSERT(__group_count_ != __invalid_count_or_rank,
|
||||
"getting group count by a unit that was not part of the parent group is not allowed");
|
||||
}
|
||||
return __group_count_;
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API unsigned group_rank() const noexcept
|
||||
{
|
||||
if constexpr (!_IsExhaustive)
|
||||
{
|
||||
_CCCL_ASSERT(is_valid(), "getting group rank of thread that is not part of the group is UB");
|
||||
}
|
||||
return __group_rank_;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr ::cuda::std::size_t static_unit_count() noexcept
|
||||
{
|
||||
return _StaticCount;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API unsigned unit_count() const noexcept
|
||||
{
|
||||
if constexpr (_StaticCount != ::cuda::std::dynamic_extent)
|
||||
{
|
||||
return static_cast<unsigned>(_StaticCount);
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (!_IsExhaustive)
|
||||
{
|
||||
_CCCL_ASSERT(is_valid(), "getting group rank of thread that is not part of the group is UB");
|
||||
}
|
||||
return __unit_count_;
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API unsigned unit_rank() const noexcept
|
||||
{
|
||||
if constexpr (!_IsExhaustive)
|
||||
{
|
||||
_CCCL_ASSERT(is_valid(), "getting unit rank of thread that is not part of the group is UB");
|
||||
}
|
||||
return __unit_rank_;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API ::cuda::device::lane_mask lane_mask() const noexcept
|
||||
{
|
||||
if constexpr (!_IsExhaustive)
|
||||
{
|
||||
_CCCL_ASSERT(is_valid(), "getting lane mask of thread that is not part of the group is UB");
|
||||
}
|
||||
return __lane_mask_;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API bool is_valid() const noexcept
|
||||
{
|
||||
if constexpr (_IsExhaustive)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
else
|
||||
{
|
||||
return __unit_rank_ != __invalid_count_or_rank;
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr bool is_always_exhaustive() noexcept
|
||||
{
|
||||
return _IsExhaustive;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr bool is_always_contiguous() noexcept
|
||||
{
|
||||
return _IsContiguous;
|
||||
}
|
||||
};
|
||||
|
||||
template <bool _IsContiguous>
|
||||
[[nodiscard]] _CCCL_DEVICE_API inline ::cuda::device::lane_mask
|
||||
__make_lane_mask_for_n(::cuda::device::lane_mask __prev_lane_mask, unsigned __n, unsigned __rank) noexcept
|
||||
{
|
||||
if constexpr (_IsContiguous)
|
||||
{
|
||||
auto __lane_mask = __prev_lane_mask;
|
||||
const auto __lane = ::cuda::ptx::get_sreg_laneid();
|
||||
|
||||
if (__lane > __rank)
|
||||
{
|
||||
__lane_mask &= ::cuda::device::lane_mask::all() << (__lane - __rank);
|
||||
}
|
||||
if (__lane + (__n - __rank) < 32)
|
||||
{
|
||||
__lane_mask &= ::cuda::device::lane_mask::all() >> (32 - __lane - (__n - __rank));
|
||||
}
|
||||
return __lane_mask;
|
||||
}
|
||||
else
|
||||
{
|
||||
auto __lane_mask = ::cuda::device::lane_mask::this_lane();
|
||||
|
||||
const auto __less_mask = __prev_lane_mask & ::cuda::device::lane_mask::all_less();
|
||||
const auto __nless = ::cuda::std::popcount(__less_mask.value());
|
||||
if (__nless > __rank)
|
||||
{
|
||||
const auto __nless_to_remove = __nless - __rank;
|
||||
const auto __last_to_remove = ::__fns(__less_mask.value(), 0, __nless_to_remove);
|
||||
__lane_mask |= ::cuda::device::lane_mask{__less_mask.value() & (~0u << (__last_to_remove + 1))};
|
||||
}
|
||||
else
|
||||
{
|
||||
__lane_mask |= __less_mask;
|
||||
}
|
||||
|
||||
const auto __greater_mask = __prev_lane_mask & ::cuda::device::lane_mask::all_greater();
|
||||
const auto __ngreater = ::cuda::std::popcount(__greater_mask.value());
|
||||
if (__rank + __ngreater >= __n)
|
||||
{
|
||||
const auto __ngreater_to_keep = __n - __rank;
|
||||
const auto __first_to_remove = ::__fns(__greater_mask.value(), 0, __ngreater_to_keep);
|
||||
__lane_mask |= ::cuda::device::lane_mask{__greater_mask.value() & ((1u << __first_to_remove) - 1u)};
|
||||
}
|
||||
else
|
||||
{
|
||||
__lane_mask |= __greater_mask;
|
||||
}
|
||||
return __lane_mask;
|
||||
}
|
||||
}
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_MAPPING_MAPPING_RESULT_CUH
|
||||
@@ -1,173 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_MAPPING_TAKE_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_MAPPING_TAKE_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__cstddef/types.h>
|
||||
#include <cuda/std/__type_traits/is_same.h>
|
||||
#include <cuda/std/__utility/cmp.h>
|
||||
#include <cuda/std/span>
|
||||
|
||||
#include <cuda/experimental/__group/fwd.cuh>
|
||||
#include <cuda/experimental/__group/mapping/mapping_result.cuh>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
template <::cuda::std::size_t _UnitCount>
|
||||
class take
|
||||
{
|
||||
static_assert(::cuda::std::in_range<unsigned>(_UnitCount), "_UnitCount must be within uint32_t range");
|
||||
|
||||
public:
|
||||
_CCCL_HIDE_FROM_ABI explicit take() = default;
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr ::cuda::std::size_t static_unit_count() noexcept
|
||||
{
|
||||
return _UnitCount;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr unsigned unit_count() const noexcept
|
||||
{
|
||||
return unsigned{_UnitCount};
|
||||
}
|
||||
|
||||
template <class _Unit, class _ParentGroup, class _PrevMappingResult>
|
||||
[[nodiscard]] _CCCL_DEVICE_API auto
|
||||
map(const _Unit&, const _ParentGroup&, const _PrevMappingResult& __prev_mapping_result) const noexcept
|
||||
{
|
||||
constexpr auto __static_prev_nunits = _PrevMappingResult::static_unit_count();
|
||||
|
||||
using _MappingResult =
|
||||
__mapping_result<_PrevMappingResult::static_group_count(),
|
||||
_UnitCount,
|
||||
_PrevMappingResult::is_always_exhaustive() && (__static_prev_nunits == _UnitCount),
|
||||
_PrevMappingResult::is_always_contiguous()>;
|
||||
|
||||
if (!__prev_mapping_result.is_valid())
|
||||
{
|
||||
return _MappingResult::invalid();
|
||||
}
|
||||
|
||||
const auto __prev_units_count = __prev_mapping_result.unit_count();
|
||||
const auto __prev_unit_rank = __prev_mapping_result.unit_rank();
|
||||
|
||||
if constexpr (__static_prev_nunits != ::cuda::std::dynamic_extent)
|
||||
{
|
||||
static_assert(__static_prev_nunits >= _UnitCount,
|
||||
"take mapping requires the previous mapping result to have at least _PrevMappingResult units");
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_ASSERT(::cuda::std::cmp_greater_equal(__prev_units_count, _UnitCount),
|
||||
"take mapping requires the previous mapping result to have at least _PrevMappingResult units");
|
||||
}
|
||||
|
||||
if (::cuda::std::cmp_greater_equal(__prev_unit_rank, static_cast<unsigned>(_UnitCount)))
|
||||
{
|
||||
return _MappingResult::invalid_with_group_count(__prev_mapping_result.group_count());
|
||||
}
|
||||
|
||||
const auto __group_count = __prev_mapping_result.group_count();
|
||||
const auto __group_rank = __prev_mapping_result.group_rank();
|
||||
const auto __unit_count = static_cast<unsigned>(_UnitCount);
|
||||
const auto __unit_rank = __prev_unit_rank;
|
||||
const auto __lane_mask =
|
||||
(::cuda::std::is_same_v<_Unit, thread_level>)
|
||||
? ::cuda::experimental::__make_lane_mask_for_n<_PrevMappingResult::is_always_contiguous()>(
|
||||
__prev_mapping_result.lane_mask(), __unit_count, __unit_rank)
|
||||
: __prev_mapping_result.lane_mask();
|
||||
return _MappingResult{__group_count, __group_rank, __unit_count, __unit_rank, __lane_mask};
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
class take<::cuda::std::dynamic_extent>
|
||||
{
|
||||
unsigned __unit_count_{0};
|
||||
|
||||
public:
|
||||
_CCCL_HIDE_FROM_ABI explicit take() = default;
|
||||
|
||||
_CCCL_DEVICE_API constexpr explicit take(unsigned __unit_count) noexcept
|
||||
: __unit_count_{__unit_count}
|
||||
{}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr ::cuda::std::size_t static_unit_count() noexcept
|
||||
{
|
||||
return ::cuda::std::dynamic_extent;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr unsigned unit_count() const noexcept
|
||||
{
|
||||
return __unit_count_;
|
||||
}
|
||||
|
||||
template <class _Unit, class _ParentGroup, class _PrevMappingResult>
|
||||
[[nodiscard]] _CCCL_DEVICE_API auto
|
||||
map(const _Unit&, const _ParentGroup&, const _PrevMappingResult& __prev_mapping_result) const noexcept
|
||||
{
|
||||
using _MappingResult =
|
||||
__mapping_result<_PrevMappingResult::static_group_count(),
|
||||
::cuda::std::dynamic_extent,
|
||||
false,
|
||||
_PrevMappingResult::is_always_contiguous()>;
|
||||
|
||||
if (!__prev_mapping_result.is_valid())
|
||||
{
|
||||
return _MappingResult::invalid();
|
||||
}
|
||||
|
||||
const auto __prev_units_count = __prev_mapping_result.unit_count();
|
||||
const auto __prev_unit_rank = __prev_mapping_result.unit_rank();
|
||||
|
||||
_CCCL_ASSERT(::cuda::std::cmp_greater_equal(__prev_units_count, __unit_count_),
|
||||
"take mapping requires the previous mapping result to have at least _PrevMappingResult units");
|
||||
|
||||
if (::cuda::std::cmp_greater_equal(__prev_unit_rank, __unit_count_))
|
||||
{
|
||||
return _MappingResult::invalid_with_group_count(__prev_mapping_result.group_count());
|
||||
}
|
||||
|
||||
const auto __group_count = __prev_mapping_result.group_count();
|
||||
const auto __group_rank = __prev_mapping_result.group_rank();
|
||||
const auto __unit_count = __unit_count_;
|
||||
const auto __unit_rank = __prev_unit_rank;
|
||||
const auto __lane_mask =
|
||||
(::cuda::std::is_same_v<_Unit, thread_level>)
|
||||
? ::cuda::experimental::__make_lane_mask_for_n<_PrevMappingResult::is_always_contiguous()>(
|
||||
__prev_mapping_result.lane_mask(), __unit_count, __unit_rank)
|
||||
: __prev_mapping_result.lane_mask();
|
||||
return _MappingResult{__group_count, __group_rank, __unit_count, __unit_rank, __lane_mask};
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES take(unsigned) -> take<::cuda::std::dynamic_extent>;
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_MAPPING_TAKE_CUH
|
||||
@@ -1,123 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_QUERIES_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_QUERIES_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__hierarchy/queries/count.h>
|
||||
#include <cuda/__hierarchy/queries/rank.h>
|
||||
#include <cuda/std/__cstddef/types.h>
|
||||
#include <cuda/std/__fwd/span.h>
|
||||
#include <cuda/std/__type_traits/is_same.h>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
template <class _Unit, class _Group>
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr ::cuda::std::size_t __static_count_query_group() noexcept
|
||||
{
|
||||
using _GroupUnit = typename _Group::unit_type;
|
||||
using _GroupMappingResult = typename _Group::__mapping_result_type;
|
||||
|
||||
constexpr auto __group_unit_count = _GroupMappingResult::static_unit_count();
|
||||
|
||||
if constexpr (::cuda::std::is_same_v<_Unit, _GroupUnit>)
|
||||
{
|
||||
return __group_unit_count;
|
||||
}
|
||||
else
|
||||
{
|
||||
using _UnitExts = decltype(_Unit::extents(_GroupUnit{}, ::cuda::std::declval<typename _Group::hierarchy_type>()));
|
||||
|
||||
if constexpr (_UnitExts::rank_dynamic() == 0 && __group_unit_count != ::cuda::std::dynamic_extent)
|
||||
{
|
||||
auto __ret = __group_unit_count;
|
||||
for (::cuda::std::size_t __i = 0; __i < _UnitExts::rank(); ++__i)
|
||||
{
|
||||
__ret *= _UnitExts::static_extent(__i);
|
||||
}
|
||||
return __ret;
|
||||
}
|
||||
else
|
||||
{
|
||||
return ::cuda::std::dynamic_extent;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Tp, class _Unit, class _Group>
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr _Tp __count_query_group(const _Group& __group) noexcept
|
||||
{
|
||||
using _GroupUnit = typename _Group::unit_type;
|
||||
|
||||
// todo(dabayer): This optimization segfaults the compiler.
|
||||
// constexpr auto __static_count = ::cuda::experimental::__static_count_query_group<_Unit, _Group>();
|
||||
// if constexpr (__static_count != ::cuda::std::dynamic_extent)
|
||||
// {
|
||||
// return static_cast<_Tp>(__static_count);
|
||||
// }
|
||||
// else
|
||||
{
|
||||
const auto __group_unit_count = static_cast<_Tp>(__group.__mapping_result().unit_count());
|
||||
if constexpr (::cuda::std::is_same_v<_Unit, _GroupUnit>)
|
||||
{
|
||||
return __group_unit_count;
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto __unit_count = __count_query<_Unit, _GroupUnit>::template __call<_Tp>(__group.hierarchy());
|
||||
return static_cast<_Tp>(__unit_count * __group_unit_count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Tp, class _Unit, class _Group>
|
||||
[[nodiscard]] _CCCL_DEVICE_API _Tp __rank_query_group(const _Group& __group) noexcept
|
||||
{
|
||||
using _GroupUnit = typename _Group::unit_type;
|
||||
|
||||
const auto __group_unit_rank = static_cast<_Tp>(__group.__mapping_result().unit_rank());
|
||||
if constexpr (::cuda::std::is_same_v<_Unit, _GroupUnit>)
|
||||
{
|
||||
return __group_unit_rank;
|
||||
}
|
||||
else
|
||||
{
|
||||
const auto __unit_rank = __rank_query<_Unit, _GroupUnit>::template __call<_Tp>(__group.hierarchy());
|
||||
const auto __unit_count = __count_query<_Unit, _GroupUnit>::template __call<_Tp>(__group.hierarchy());
|
||||
return static_cast<_Tp>(__group_unit_rank * __unit_count + __unit_rank);
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Unit, class _Group>
|
||||
[[nodiscard]] _CCCL_DEVICE_API bool __is_part_of_group(const _Group& __group) noexcept
|
||||
{
|
||||
return __group.__mapping_result().is_valid();
|
||||
}
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_QUERIES_CUH
|
||||
@@ -1,161 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_SYNCHRONIZER_BARRIER_SYNCHRONIZER_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_SYNCHRONIZER_BARRIER_SYNCHRONIZER_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/barrier>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/std/__cstddef/types.h>
|
||||
#include <cuda/std/__type_traits/is_same.h>
|
||||
#include <cuda/std/__utility/declval.h>
|
||||
#include <cuda/std/span>
|
||||
|
||||
#include <cuda/experimental/__group/concepts.cuh>
|
||||
#include <cuda/experimental/__group/fwd.cuh>
|
||||
#include <cuda/experimental/__group/traits.cuh>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
template <class _Level>
|
||||
[[nodiscard]] _CCCL_DEVICE_API _CCCL_CONSTEVAL thread_scope __minimum_required_scope_for() noexcept
|
||||
{
|
||||
if constexpr (::cuda::std::is_same_v<_Level, thread_level>)
|
||||
{
|
||||
return thread_scope_thread;
|
||||
}
|
||||
else if constexpr (::cuda::std::is_same_v<_Level, warp_level> || ::cuda::std::is_same_v<_Level, block_level>)
|
||||
{
|
||||
return thread_scope_block;
|
||||
}
|
||||
else if constexpr (::cuda::std::is_same_v<_Level, cluster_level> || ::cuda::std::is_same_v<_Level, grid_level>)
|
||||
{
|
||||
return thread_scope_device;
|
||||
}
|
||||
else
|
||||
{
|
||||
return thread_scope_system;
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Tp>
|
||||
inline constexpr thread_scope __barrier_scope_v = thread_scope_system;
|
||||
template <thread_scope _Sco, class _ComplFn>
|
||||
inline constexpr thread_scope __barrier_scope_v<barrier<_Sco, _ComplFn>> = _Sco;
|
||||
|
||||
template <class _Barrier, ::cuda::std::size_t _Np>
|
||||
class barrier_synchronizer
|
||||
{
|
||||
static_assert(__is_cuda_barrier_v<_Barrier>, "_Barrier must be cv-unqualified cuda::barrier type");
|
||||
|
||||
::cuda::std::span<_Barrier, _Np> __barriers_;
|
||||
|
||||
public:
|
||||
using barrier_type = _Barrier;
|
||||
|
||||
struct __synchronizer_instance
|
||||
{
|
||||
template <class _MappingResult>
|
||||
_CCCL_DEVICE_API void
|
||||
do_sync(const _MappingResult& __mapping_result, const barrier_synchronizer& __synchronizer) const noexcept
|
||||
{
|
||||
__synchronizer.__barriers_[__mapping_result.group_rank()].arrive_and_wait();
|
||||
}
|
||||
|
||||
template <class _MappingResult>
|
||||
_CCCL_DEVICE_API void
|
||||
do_sync_aligned(const _MappingResult& __mapping_result, const barrier_synchronizer& __synchronizer) const noexcept
|
||||
{
|
||||
__synchronizer.__barriers_[__mapping_result.group_rank()].arrive_and_wait();
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_DEVICE_API barrier_synchronizer(::cuda::std::span<_Barrier, _Np> __barriers) noexcept
|
||||
: __barriers_(__barriers)
|
||||
{}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API ::cuda::std::span<_Barrier, _Np> barriers() const noexcept
|
||||
{
|
||||
return __barriers_;
|
||||
}
|
||||
|
||||
template <class _Unit, class _ParentGroup, class _Mapping, class _MappingResult>
|
||||
[[nodiscard]] _CCCL_DEVICE_API __synchronizer_instance make_instance(
|
||||
const _Unit&,
|
||||
const _ParentGroup& __parent,
|
||||
const _Mapping& __mapping,
|
||||
const _MappingResult& __mapping_result) const noexcept
|
||||
{
|
||||
using _Level = typename _ParentGroup::level_type;
|
||||
|
||||
// todo(dabayer): Relax this condition if all units in the group are within a level that is smaller than _Level.
|
||||
static_assert(__barrier_scope_v<_Barrier> <= ::cuda::experimental::__minimum_required_scope_for<_Level>(),
|
||||
"_Barrier's thread scope is insufficient for group synchronization in _Level");
|
||||
|
||||
if constexpr (_MappingResult::static_group_count() != ::cuda::std::dynamic_extent
|
||||
&& _Np != ::cuda::std::dynamic_extent)
|
||||
{
|
||||
static_assert(_MappingResult::static_group_count() <= _Np, "invalid number of barriers passed");
|
||||
}
|
||||
else
|
||||
{
|
||||
_CCCL_ASSERT(__mapping_result.group_count() <= __barriers_.size(), "invalid number of barriers passed");
|
||||
}
|
||||
|
||||
::cuda::std::size_t __nthread_in_unit = 1;
|
||||
::cuda::std::size_t __thread_rank_in_unit = 0;
|
||||
if constexpr (!::cuda::std::is_same_v<thread_level, _Unit>)
|
||||
{
|
||||
__nthread_in_unit = gpu_thread.count(_Unit{}, __parent.hierarchy());
|
||||
__thread_rank_in_unit = gpu_thread.rank(_Unit{}, __parent.hierarchy());
|
||||
}
|
||||
|
||||
if (__mapping_result.is_valid() && __mapping_result.unit_rank() == 0 && __thread_rank_in_unit == 0)
|
||||
{
|
||||
init(&__barriers_[__mapping_result.group_rank()],
|
||||
static_cast<::cuda::std::ptrdiff_t>(__mapping_result.unit_count() * __nthread_in_unit));
|
||||
}
|
||||
|
||||
// todo(dabayer): How we can expose making this aligned?
|
||||
__parent.sync();
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
template <class _Barrier, ::cuda::std::size_t _Np>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES barrier_synchronizer(::cuda::std::span<_Barrier, _Np>)
|
||||
-> barrier_synchronizer<_Barrier, _Np>;
|
||||
|
||||
_CCCL_TEMPLATE(class _Tp)
|
||||
_CCCL_REQUIRES(__is_spannable<_Tp&> _CCCL_AND(!::cuda::std::__is_cuda_std_span_v<::cuda::std::remove_cv_t<_Tp>>))
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES barrier_synchronizer(_Tp&)
|
||||
-> barrier_synchronizer<_SpanElementType<decltype(::cuda::std::span(::cuda::std::declval<_Tp&>()))>,
|
||||
decltype(::cuda::std::span(::cuda::std::declval<_Tp&>()))::extent>;
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_SYNCHRONIZER_BARRIER_SYNCHRONIZER_CUH
|
||||
@@ -1,93 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_SYNCHRONIZER_LANE_SYNCHRONIZER_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_SYNCHRONIZER_LANE_SYNCHRONIZER_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__cmath/pow2.h>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/std/__bit/popcount.h>
|
||||
#include <cuda/std/__cstddef/types.h>
|
||||
#include <cuda/std/__fwd/span.h>
|
||||
#include <cuda/std/__type_traits/is_same.h>
|
||||
|
||||
#include <cuda/experimental/__group/concepts.cuh>
|
||||
#include <cuda/experimental/__group/fwd.cuh>
|
||||
#include <cuda/experimental/__group/mapping/group_by.cuh>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
class lane_synchronizer
|
||||
{
|
||||
template <class _Level, class _Tp>
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr bool __is_supported_count(_Tp __n) noexcept
|
||||
{
|
||||
return (::cuda::is_power_of_two(__n) || ::cuda::std::is_same_v<_Level, warp_level>) && __n <= 32;
|
||||
}
|
||||
|
||||
public:
|
||||
struct __synchronizer_instance
|
||||
{
|
||||
[[nodiscard]] _CCCL_DEVICE_API static __synchronizer_instance invalid() noexcept
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
template <class _MappingResult>
|
||||
_CCCL_DEVICE_API void do_sync(const _MappingResult& __mapping_result, const lane_synchronizer&) const noexcept
|
||||
{
|
||||
::__syncwarp(__mapping_result.lane_mask().value());
|
||||
}
|
||||
|
||||
template <class _MappingResult>
|
||||
_CCCL_DEVICE_API void
|
||||
do_sync_aligned(const _MappingResult& __mapping_result, const lane_synchronizer&) const noexcept
|
||||
{
|
||||
::__syncwarp(__mapping_result.lane_mask().value());
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_HIDE_FROM_ABI explicit lane_synchronizer() = default;
|
||||
|
||||
template <class _Unit, class _ParentGroup, class _Mapping, class _MappingResult>
|
||||
[[nodiscard]] _CCCL_DEVICE_API __synchronizer_instance make_instance(
|
||||
const _Unit&, const _ParentGroup&, const _Mapping&, const _MappingResult& __mapping_result) const noexcept
|
||||
{
|
||||
static_assert(::cuda::std::is_same_v<_Unit, thread_level>, "_Unit must be cuda::thread_level");
|
||||
static_assert(__group_mapping_result<_MappingResult>);
|
||||
if (__mapping_result.is_valid())
|
||||
{
|
||||
_CCCL_ASSERT(::cuda::std::popcount(__mapping_result.lane_mask().value()) == __mapping_result.unit_count(),
|
||||
"lane_synchronizer can only synchronize units within the same warp");
|
||||
}
|
||||
return {};
|
||||
}
|
||||
};
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_SYNCHRONIZER_LANE_SYNCHRONIZER_CUH
|
||||
@@ -1,600 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_THIS_GROUP_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_THIS_GROUP_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__warp/lane_mask.h>
|
||||
#include <cuda/hierarchy>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__limits/numeric_limits.h>
|
||||
#include <cuda/std/__type_traits/is_integer.h>
|
||||
#include <cuda/std/__type_traits/is_same.h>
|
||||
|
||||
#include <cuda/experimental/__group/fwd.cuh>
|
||||
#include <cuda/experimental/__group/implicit_hierarchy.cuh>
|
||||
|
||||
#if _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
# include <cooperative_groups.h>
|
||||
#endif // _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
template <class _HierarchyLike>
|
||||
using __hierarchy_type_of =
|
||||
::cuda::std::remove_cvref_t<decltype(::cuda::__unpack_hierarchy_if_needed(::cuda::std::declval<_HierarchyLike>()))>;
|
||||
|
||||
# if _CCCL_CUDA_COMPILATION()
|
||||
template <bool _Aligned>
|
||||
_CCCL_DEVICE_API void __block_sync() noexcept
|
||||
{
|
||||
if constexpr (_Aligned)
|
||||
{
|
||||
::__syncthreads();
|
||||
}
|
||||
else
|
||||
{
|
||||
::__barrier_sync(0);
|
||||
}
|
||||
}
|
||||
|
||||
template <bool _Aligned>
|
||||
_CCCL_DEVICE_API void __cluster_sync() noexcept
|
||||
{
|
||||
NV_IF_ELSE_TARGET(NV_PROVIDES_SM_90,
|
||||
({
|
||||
if constexpr (_Aligned)
|
||||
{
|
||||
asm volatile("barrier.cluster.arrive.aligned;");
|
||||
asm volatile("barrier.cluster.wait.aligned;");
|
||||
}
|
||||
else
|
||||
{
|
||||
::__cluster_barrier_arrive();
|
||||
::__cluster_barrier_wait();
|
||||
}
|
||||
}),
|
||||
({ ::cuda::experimental::__block_sync<_Aligned>(); }))
|
||||
}
|
||||
# endif // _CCCL_CUDA_COMPILATION()
|
||||
|
||||
template <class _Level>
|
||||
struct __this_mapping_result
|
||||
{
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr ::cuda::std::size_t static_group_count() noexcept
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API unsigned group_count() const noexcept
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API unsigned group_rank() const noexcept
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr ::cuda::std::size_t static_unit_count() noexcept
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API unsigned unit_count() const noexcept
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API unsigned unit_rank() const noexcept
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API ::cuda::device::lane_mask lane_mask() const noexcept
|
||||
{
|
||||
if constexpr (::cuda::std::is_same_v<_Level, thread_level>)
|
||||
{
|
||||
return ::cuda::device::lane_mask::this_lane();
|
||||
}
|
||||
else
|
||||
{
|
||||
return ::cuda::device::lane_mask::all();
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API bool is_valid() const noexcept
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr bool is_always_exhaustive() noexcept
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API static constexpr bool is_always_contiguous() noexcept
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// todo(dabayer): add method that determines whether the unit is part of the group or not.
|
||||
};
|
||||
|
||||
// todo: use __hier_ in queries
|
||||
template <class _Level, class _Hierarchy>
|
||||
class __this_group_base
|
||||
{
|
||||
static_assert(__is_hierarchy_level_v<_Level>);
|
||||
static_assert(__is_hierarchy_v<_Hierarchy>);
|
||||
|
||||
protected:
|
||||
using __mapping_result_type = __this_mapping_result<_Level>;
|
||||
|
||||
_Hierarchy __hier_;
|
||||
|
||||
public:
|
||||
_CCCL_DEVICE_API explicit __this_group_base() noexcept
|
||||
: __hier_{::cuda::experimental::__implicit_hierarchy()}
|
||||
{}
|
||||
|
||||
_CCCL_TEMPLATE(class _HierarchyLike)
|
||||
_CCCL_REQUIRES(::cuda::std::is_same_v<_Hierarchy, __hierarchy_type_of<_HierarchyLike>>)
|
||||
_CCCL_DEVICE_API __this_group_base(const _HierarchyLike& __hier_like) noexcept
|
||||
: __hier_{::cuda::__unpack_hierarchy_if_needed(__hier_like)}
|
||||
{}
|
||||
|
||||
_CCCL_TEMPLATE(class _Tp, class _InLevel, class _Level2 = _Level)
|
||||
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp> _CCCL_AND __is_hierarchy_level_v<_InLevel> _CCCL_AND(
|
||||
!::cuda::std::is_same_v<_Level2, grid_level>))
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr _Tp count_as(const _InLevel& __in_level) const noexcept
|
||||
{
|
||||
return _Level{}.template count_as<_Tp>(__in_level, __hier_);
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _InLevel, class _Level2 = _Level)
|
||||
_CCCL_REQUIRES(__is_hierarchy_level_v<_InLevel> _CCCL_AND(!::cuda::std::is_same_v<_Level2, grid_level>))
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr auto count(const _InLevel& __in_level) const noexcept
|
||||
{
|
||||
return _Level{}.count(__in_level, __hier_);
|
||||
}
|
||||
|
||||
# if _CCCL_CUDA_COMPILATION()
|
||||
_CCCL_TEMPLATE(class _Tp, class _InLevel, class _Level2 = _Level)
|
||||
_CCCL_REQUIRES(::cuda::std::__cccl_is_integer_v<_Tp> _CCCL_AND __is_hierarchy_level_v<_InLevel> _CCCL_AND(
|
||||
!::cuda::std::is_same_v<_Level2, grid_level>))
|
||||
[[nodiscard]] _CCCL_DEVICE_API _Tp rank_as(const _InLevel& __in_level) const noexcept
|
||||
{
|
||||
return _Level{}.template rank_as<_Tp>(__in_level, __hier_);
|
||||
}
|
||||
|
||||
_CCCL_TEMPLATE(class _InLevel, class _Level2 = _Level)
|
||||
_CCCL_REQUIRES(__is_hierarchy_level_v<_InLevel> _CCCL_AND(!::cuda::std::is_same_v<_Level2, grid_level>))
|
||||
[[nodiscard]] _CCCL_DEVICE_API auto rank(const _InLevel& __in_level) const noexcept
|
||||
{
|
||||
return _Level{}.rank(__in_level, __hier_);
|
||||
}
|
||||
# endif // _CCCL_CUDA_COMPILATION()
|
||||
};
|
||||
|
||||
template <class _Hierarchy>
|
||||
class this_thread : __this_group_base<thread_level, _Hierarchy>
|
||||
{
|
||||
using __base_type = __this_group_base<thread_level, _Hierarchy>;
|
||||
|
||||
public:
|
||||
using unit_type = thread_level;
|
||||
using level_type = thread_level;
|
||||
using mapping_type = void;
|
||||
using typename __base_type::__mapping_result_type;
|
||||
using hierarchy_type = _Hierarchy;
|
||||
using synchronizer_type = void;
|
||||
|
||||
using __base_type::__base_type;
|
||||
using __base_type::count;
|
||||
using __base_type::count_as;
|
||||
# if _CCCL_CUDA_COMPILATION()
|
||||
using __base_type::rank;
|
||||
using __base_type::rank_as;
|
||||
|
||||
# if _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
template <class _Parent>
|
||||
_CCCL_DEVICE_API this_thread(const ::cooperative_groups::thread_block_tile<1, _Parent>&) noexcept
|
||||
{}
|
||||
# endif // _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
|
||||
_CCCL_DEVICE_API void sync() const noexcept {}
|
||||
|
||||
_CCCL_DEVICE_API void sync_aligned() const noexcept {}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr __mapping_result_type __mapping_result() const noexcept
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr const _Hierarchy& hierarchy() const noexcept
|
||||
{
|
||||
return __base_type::__hier_;
|
||||
}
|
||||
# endif // _CCCL_CUDA_COMPILATION()
|
||||
};
|
||||
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_thread() -> this_thread<__implicit_hierarchy_t>;
|
||||
|
||||
_CCCL_TEMPLATE(class _Hierarchy)
|
||||
_CCCL_REQUIRES(__is_or_has_hierarchy_member_v<_Hierarchy>)
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_thread(const _Hierarchy&) -> this_thread<__hierarchy_type_of<_Hierarchy>>;
|
||||
|
||||
# if _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_thread(const ::cooperative_groups::thread_block_tile<1, void>&)
|
||||
-> this_thread<__implicit_hierarchy_t>;
|
||||
# endif // _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
|
||||
template <class _Hierarchy>
|
||||
class this_warp : __this_group_base<warp_level, _Hierarchy>
|
||||
{
|
||||
using __base_type = __this_group_base<warp_level, _Hierarchy>;
|
||||
|
||||
public:
|
||||
using unit_type = warp_level;
|
||||
using level_type = warp_level;
|
||||
using mapping_type = void;
|
||||
using typename __base_type::__mapping_result_type;
|
||||
using hierarchy_type = _Hierarchy;
|
||||
using synchronizer_type = void;
|
||||
|
||||
using __base_type::__base_type;
|
||||
using __base_type::count;
|
||||
using __base_type::count_as;
|
||||
# if _CCCL_CUDA_COMPILATION()
|
||||
using __base_type::rank;
|
||||
using __base_type::rank_as;
|
||||
|
||||
# if _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
template <class _Parent>
|
||||
_CCCL_DEVICE_API this_warp(const ::cooperative_groups::thread_block_tile<32, _Parent>&) noexcept
|
||||
{}
|
||||
# endif // _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
|
||||
_CCCL_DEVICE_API void sync() const noexcept
|
||||
{
|
||||
::__syncwarp();
|
||||
}
|
||||
|
||||
_CCCL_DEVICE_API void sync_aligned() const noexcept
|
||||
{
|
||||
::__syncwarp();
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr __mapping_result_type __mapping_result() const noexcept
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr const _Hierarchy& hierarchy() const noexcept
|
||||
{
|
||||
return __base_type::__hier_;
|
||||
}
|
||||
# endif // _CCCL_CUDA_COMPILATION()
|
||||
};
|
||||
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_warp() -> this_warp<__implicit_hierarchy_t>;
|
||||
|
||||
_CCCL_TEMPLATE(class _Hierarchy)
|
||||
_CCCL_REQUIRES(__is_or_has_hierarchy_member_v<_Hierarchy>)
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_warp(const _Hierarchy&) -> this_warp<__hierarchy_type_of<_Hierarchy>>;
|
||||
|
||||
# if _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
template <class _Parent>
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_warp(const ::cooperative_groups::thread_block_tile<32, _Parent>&)
|
||||
-> this_warp<__implicit_hierarchy_t>;
|
||||
# endif // _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
|
||||
template <class _Hierarchy>
|
||||
class this_block : __this_group_base<block_level, _Hierarchy>
|
||||
{
|
||||
using __base_type = __this_group_base<block_level, _Hierarchy>;
|
||||
|
||||
public:
|
||||
using unit_type = block_level;
|
||||
using level_type = block_level;
|
||||
using mapping_type = void;
|
||||
using typename __base_type::__mapping_result_type;
|
||||
using hierarchy_type = _Hierarchy;
|
||||
using synchronizer_type = void;
|
||||
|
||||
using __base_type::__base_type;
|
||||
using __base_type::count;
|
||||
using __base_type::count_as;
|
||||
|
||||
# if _CCCL_CUDA_COMPILATION()
|
||||
using __base_type::rank;
|
||||
using __base_type::rank_as;
|
||||
|
||||
# if _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
_CCCL_DEVICE_API this_block(const ::cooperative_groups::thread_block&) noexcept {}
|
||||
# endif // _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
|
||||
_CCCL_DEVICE_API void sync() const noexcept
|
||||
{
|
||||
::cuda::experimental::__block_sync<false>();
|
||||
}
|
||||
|
||||
_CCCL_DEVICE_API void sync_aligned() const noexcept
|
||||
{
|
||||
::cuda::experimental::__block_sync<true>();
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr __mapping_result_type __mapping_result() const noexcept
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr const _Hierarchy& hierarchy() const noexcept
|
||||
{
|
||||
return __base_type::__hier_;
|
||||
}
|
||||
# endif // _CCCL_CUDA_COMPILATION()
|
||||
};
|
||||
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_block() -> this_block<__implicit_hierarchy_t>;
|
||||
|
||||
_CCCL_TEMPLATE(class _Hierarchy)
|
||||
_CCCL_REQUIRES(__is_or_has_hierarchy_member_v<_Hierarchy>)
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_block(const _Hierarchy&) -> this_block<__hierarchy_type_of<_Hierarchy>>;
|
||||
|
||||
# if _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_block(const ::cooperative_groups::thread_block&)
|
||||
-> this_block<__implicit_hierarchy_t>;
|
||||
# endif // _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
|
||||
template <class _Hierarchy>
|
||||
class this_cluster : __this_group_base<cluster_level, _Hierarchy>
|
||||
{
|
||||
using __base_type = __this_group_base<cluster_level, _Hierarchy>;
|
||||
|
||||
public:
|
||||
using unit_type = cluster_level;
|
||||
using level_type = cluster_level;
|
||||
using mapping_type = void;
|
||||
using typename __base_type::__mapping_result_type;
|
||||
using hierarchy_type = _Hierarchy;
|
||||
using synchronizer_type = void;
|
||||
|
||||
using __base_type::__base_type;
|
||||
using __base_type::count;
|
||||
using __base_type::count_as;
|
||||
|
||||
# if _CCCL_CUDA_COMPILATION()
|
||||
using __base_type::rank;
|
||||
using __base_type::rank_as;
|
||||
|
||||
# if _CCCL_HAS_COOPERATIVE_GROUPS() && defined(_CG_HAS_CLUSTER_GROUP)
|
||||
_CCCL_DEVICE_API this_cluster(const ::cooperative_groups::cluster_group&) noexcept {}
|
||||
# endif // _CCCL_HAS_COOPERATIVE_GROUPS() && defined(_CG_HAS_CLUSTER_GROUP)
|
||||
|
||||
_CCCL_DEVICE_API void sync() const noexcept
|
||||
{
|
||||
if constexpr (_Hierarchy::has_level(cluster))
|
||||
{
|
||||
::cuda::experimental::__cluster_sync<false>();
|
||||
}
|
||||
else
|
||||
{
|
||||
::cuda::experimental::__block_sync<false>();
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_DEVICE_API void sync_aligned() const noexcept
|
||||
{
|
||||
if constexpr (_Hierarchy::has_level(cluster))
|
||||
{
|
||||
::cuda::experimental::__cluster_sync<true>();
|
||||
}
|
||||
else
|
||||
{
|
||||
::cuda::experimental::__block_sync<true>();
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr __mapping_result_type __mapping_result() const noexcept
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr const _Hierarchy& hierarchy() const noexcept
|
||||
{
|
||||
return __base_type::__hier_;
|
||||
}
|
||||
# endif // _CCCL_CUDA_COMPILATION()
|
||||
};
|
||||
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_cluster() -> this_cluster<__implicit_hierarchy_t>;
|
||||
|
||||
_CCCL_TEMPLATE(class _Hierarchy)
|
||||
_CCCL_REQUIRES(__is_or_has_hierarchy_member_v<_Hierarchy>)
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_cluster(const _Hierarchy&) -> this_cluster<__hierarchy_type_of<_Hierarchy>>;
|
||||
|
||||
# if _CCCL_HAS_COOPERATIVE_GROUPS() && defined(_CG_HAS_CLUSTER_GROUP)
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_cluster(const ::cooperative_groups::cluster_group&)
|
||||
-> this_cluster<__implicit_hierarchy_t>;
|
||||
# endif // _CCCL_HAS_COOPERATIVE_GROUPS() && defined(_CG_HAS_CLUSTER_GROUP)
|
||||
|
||||
// Synchronizing whole grid requires driver support and the kernel must be launched using the cooperative launch API.
|
||||
// This part is extracted from grid synchronization implementation in cooperative groups.
|
||||
# if _CCCL_CUDA_COMPILATION()
|
||||
[[nodiscard]] _CCCL_DEVICE_API inline unsigned* __get_grid_barrier_ptr() noexcept
|
||||
{
|
||||
struct __grid_workspace
|
||||
{
|
||||
unsigned __size_;
|
||||
unsigned __barrier_;
|
||||
};
|
||||
|
||||
__grid_workspace* __grid_workspace_ptr;
|
||||
asm("mov.b64 %0, {%%envreg2, %%envreg1};" : "=l"(__grid_workspace_ptr));
|
||||
_CCCL_ASSERT(__grid_workspace_ptr != nullptr,
|
||||
"Synchronizing grid requires the kernel to be launched using the cooperative launch.");
|
||||
|
||||
return &__grid_workspace_ptr->__barrier_;
|
||||
}
|
||||
# endif // _CCCL_CUDA_COMPILATION()
|
||||
|
||||
template <class _Hierarchy>
|
||||
class this_grid : __this_group_base<grid_level, _Hierarchy>
|
||||
{
|
||||
using __base_type = __this_group_base<grid_level, _Hierarchy>;
|
||||
|
||||
# if _CCCL_CUDA_COMPILATION()
|
||||
template <bool _Aligned>
|
||||
_CCCL_DEVICE_API void __sync_impl() const noexcept
|
||||
{
|
||||
const auto __barrier_ptr = ::cuda::experimental::__get_grid_barrier_ptr();
|
||||
|
||||
// Synchronize the block before synchronizing with the other blocks.
|
||||
::cuda::experimental::__block_sync<_Aligned>();
|
||||
|
||||
// Synchronize with other blocks using the thread 0 in block.
|
||||
const auto __thread_idx = gpu_thread.index(block, hierarchy());
|
||||
if ((__thread_idx.x | __thread_idx.y | __thread_idx.z) == 0)
|
||||
{
|
||||
const auto __expected = block.count_as<unsigned>(grid, hierarchy());
|
||||
unsigned __nblocks = 1;
|
||||
|
||||
const auto __block_idx = block.index(grid, hierarchy());
|
||||
if ((__block_idx.x | __block_idx.y | __block_idx.z) == 0)
|
||||
{
|
||||
__nblocks = unsigned{::cuda::std::numeric_limits<int>::min()} - (__expected - 1);
|
||||
}
|
||||
|
||||
unsigned __old_barrier_value;
|
||||
# if _CCCL_HAS_NV_ATOMIC_BUILTINS()
|
||||
__old_barrier_value =
|
||||
__nv_atomic_fetch_add(__barrier_ptr, __nblocks, __NV_ATOMIC_RELEASE, __NV_THREAD_SCOPE_DEVICE);
|
||||
# else // ^^^ _CCCL_HAS_NV_ATOMIC_BUILTINS() ^^^ / vvv !_CCCL_HAS_NV_ATOMIC_BUILTINS() vvv
|
||||
asm volatile("atom.add.release.gpu.u32 %0, [%1], %2;"
|
||||
: "=r"(__old_barrier_value)
|
||||
: "l"(__barrier_ptr), "r"(__nblocks)
|
||||
: "memory");
|
||||
# endif // ^^^ !_CCCL_HAS_NV_ATOMIC_BUILTINS() ^^^
|
||||
unsigned __curr_barrier_value;
|
||||
do
|
||||
{
|
||||
# if _CCCL_HAS_NV_ATOMIC_BUILTINS()
|
||||
__nv_atomic_load(__barrier_ptr, &__curr_barrier_value, __NV_ATOMIC_ACQUIRE, __NV_THREAD_SCOPE_DEVICE);
|
||||
# else // ^^^ _CCCL_HAS_NV_ATOMIC_BUILTINS() ^^^ / vvv !_CCCL_HAS_NV_ATOMIC_BUILTINS() vvv
|
||||
asm volatile("ld.acquire.gpu.u32 %0, [%1];" : "=r"(__curr_barrier_value) : "l"(__barrier_ptr) : "memory");
|
||||
# endif // ^^^ !_CCCL_HAS_NV_ATOMIC_BUILTINS() ^^^
|
||||
} while (static_cast<int>(__old_barrier_value) < 0 == static_cast<int>(__curr_barrier_value) < 0);
|
||||
}
|
||||
|
||||
// Wait for the thread 0 to finish the inter block synchronization.
|
||||
::cuda::experimental::__block_sync<_Aligned>();
|
||||
}
|
||||
# endif // _CCCL_CUDA_COMPILATION()
|
||||
|
||||
public:
|
||||
using unit_type = grid_level;
|
||||
using level_type = grid_level;
|
||||
using mapping_type = void;
|
||||
using typename __base_type::__mapping_result_type;
|
||||
using hierarchy_type = _Hierarchy;
|
||||
using synchronizer_type = void;
|
||||
|
||||
using __base_type::__base_type;
|
||||
|
||||
# if _CCCL_CUDA_COMPILATION()
|
||||
# if _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
_CCCL_DEVICE_API this_grid(const ::cooperative_groups::grid_group&) noexcept {}
|
||||
# endif // _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
|
||||
_CCCL_DEVICE_API void sync() const noexcept
|
||||
{
|
||||
__sync_impl<false>();
|
||||
}
|
||||
|
||||
_CCCL_DEVICE_API void sync_aligned() const noexcept
|
||||
{
|
||||
__sync_impl<true>();
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr __mapping_result_type __mapping_result() const noexcept
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_DEVICE_API constexpr const _Hierarchy& hierarchy() const noexcept
|
||||
{
|
||||
return __base_type::__hier_;
|
||||
}
|
||||
# endif // _CCCL_CUDA_COMPILATION()
|
||||
};
|
||||
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_grid() -> this_grid<__implicit_hierarchy_t>;
|
||||
|
||||
_CCCL_TEMPLATE(class _Hierarchy)
|
||||
_CCCL_REQUIRES(__is_or_has_hierarchy_member_v<_Hierarchy>)
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_grid(const _Hierarchy&) -> this_grid<__hierarchy_type_of<_Hierarchy>>;
|
||||
|
||||
# if _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
_CCCL_DEDUCTION_GUIDE_ATTRIBUTES this_grid(const ::cooperative_groups::grid_group&)
|
||||
-> this_grid<__implicit_hierarchy_t>;
|
||||
# endif // _CCCL_HAS_COOPERATIVE_GROUPS()
|
||||
|
||||
_CCCL_TEMPLATE(class _Level, class... _Args)
|
||||
_CCCL_REQUIRES(__is_hierarchy_level_v<_Level>)
|
||||
[[nodiscard]] _CCCL_DEVICE_API auto make_this_group(const _Level&, _Args&&... __args) noexcept
|
||||
{
|
||||
if constexpr (::cuda::std::is_same_v<_Level, thread_level>)
|
||||
{
|
||||
return this_thread{::cuda::std::forward<_Args>(__args)...};
|
||||
}
|
||||
else if constexpr (::cuda::std::is_same_v<_Level, warp_level>)
|
||||
{
|
||||
return this_warp{::cuda::std::forward<_Args>(__args)...};
|
||||
}
|
||||
else if constexpr (::cuda::std::is_same_v<_Level, block_level>)
|
||||
{
|
||||
return this_block{::cuda::std::forward<_Args>(__args)...};
|
||||
}
|
||||
else if constexpr (::cuda::std::is_same_v<_Level, cluster_level>)
|
||||
{
|
||||
return this_cluster{::cuda::std::forward<_Args>(__args)...};
|
||||
}
|
||||
else if constexpr (::cuda::std::is_same_v<_Level, grid_level>)
|
||||
{
|
||||
return this_grid{::cuda::std::forward<_Args>(__args)...};
|
||||
}
|
||||
else
|
||||
{
|
||||
static_assert(::cuda::std::__always_false_v<_Level>, "unknown _Level");
|
||||
}
|
||||
}
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_THIS_GROUP_CUH
|
||||
@@ -1,62 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA_EXPERIMENTAL___GROUP_TRAITS_CUH
|
||||
#define _CUDA_EXPERIMENTAL___GROUP_TRAITS_CUH
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__type_traits/void_t.h>
|
||||
#include <cuda/std/__utility/declval.h>
|
||||
#include <cuda/std/span>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
#if !defined(_CCCL_DOXYGEN_INVOKED)
|
||||
|
||||
namespace cuda::experimental
|
||||
{
|
||||
template <class _Mapping, class _Unit, class _ParentGroup>
|
||||
using __group_mapping_result_t = decltype(::cuda::std::declval<_Mapping>().map(
|
||||
::cuda::std::declval<_Unit>(), ::cuda::std::declval<const _ParentGroup&>()));
|
||||
|
||||
template <class _Synchronizer, class _Unit, class _ParentGroup, class _Mapping, class _MappingResult>
|
||||
using __group_synchronizer_instance_t = decltype(::cuda::std::declval<_Synchronizer>().make_instance(
|
||||
::cuda::std::declval<const _Unit&>(),
|
||||
::cuda::std::declval<const _ParentGroup&>(),
|
||||
::cuda::std::declval<const _Mapping&>(),
|
||||
::cuda::std::declval<const _MappingResult&>()));
|
||||
|
||||
template <class _Tp, class = void>
|
||||
inline constexpr bool __is_spannable = false;
|
||||
template <class _Tp>
|
||||
inline constexpr bool
|
||||
__is_spannable<_Tp, ::cuda::std::void_t<decltype(::cuda::std::span(::cuda::std::declval<_Tp>()))>> = true;
|
||||
|
||||
template <class _Span>
|
||||
using _SpanElementType = typename _Span::element_type;
|
||||
|
||||
template <class _Span>
|
||||
using _SpanValueType = typename _Span::value_type;
|
||||
} // namespace cuda::experimental
|
||||
|
||||
#endif // !_CCCL_DOXYGEN_INVOKED
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CUDA_EXPERIMENTAL___GROUP_TRAITS_CUH
|
||||
Reference in New Issue
Block a user