[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
EngineX CI
818
cccl_upstream/libcudacxx/include/cuda/__launch/configuration.h
Normal file
818
cccl_upstream/libcudacxx/include/cuda/__launch/configuration.h
Normal file
@@ -0,0 +1,818 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA___LAUNCH_CONFIGURATION_H
|
||||
#define _CUDA___LAUNCH_CONFIGURATION_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
# include <cuda/__driver/driver_api.h>
|
||||
# include <cuda/__hierarchy/hierarchy_dimensions.h>
|
||||
# include <cuda/__numeric/overflow_cast.h>
|
||||
# include <cuda/__ptx/instructions/get_sreg.h>
|
||||
# include <cuda/std/__cstddef/types.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__host_stdlib/stdexcept>
|
||||
# include <cuda/std/__type_traits/is_const.h>
|
||||
# include <cuda/std/__type_traits/is_reference.h>
|
||||
# include <cuda/std/__type_traits/is_unbounded_array.h>
|
||||
# include <cuda/std/__type_traits/rank.h>
|
||||
# include <cuda/std/span>
|
||||
# include <cuda/std/tuple>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA
|
||||
|
||||
template <typename Dimensions, typename... Options>
|
||||
struct kernel_config;
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
struct launch_option
|
||||
{
|
||||
static constexpr bool needs_attribute_space = false;
|
||||
static constexpr bool is_relevant_on_device = false;
|
||||
};
|
||||
|
||||
// Might need to go to the main namespace?
|
||||
enum class launch_option_kind
|
||||
{
|
||||
cooperative_launch,
|
||||
dynamic_shared_memory,
|
||||
launch_priority
|
||||
};
|
||||
|
||||
struct option_not_found
|
||||
{};
|
||||
|
||||
template <__detail::launch_option_kind Kind>
|
||||
struct find_option_in_tuple_impl
|
||||
{
|
||||
template <typename Option, typename... Options>
|
||||
_CCCL_DEVICE_API auto& operator()(const Option& opt, const Options&... rest)
|
||||
{
|
||||
if constexpr (Option::kind == Kind)
|
||||
{
|
||||
return opt;
|
||||
}
|
||||
else
|
||||
{
|
||||
return (*this)(rest...);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_DEVICE_API auto operator()()
|
||||
{
|
||||
return option_not_found();
|
||||
}
|
||||
};
|
||||
|
||||
template <__detail::launch_option_kind Kind, typename... Options>
|
||||
_CCCL_DEVICE_API auto& find_option_in_tuple(const ::cuda::std::tuple<Options...>& tuple)
|
||||
{
|
||||
return ::cuda::std::apply(find_option_in_tuple_impl<Kind>(), tuple);
|
||||
}
|
||||
|
||||
template <typename _Option, typename... _OptionsList>
|
||||
inline constexpr bool __option_present_in_list = ((_Option::kind == _OptionsList::kind) || ...);
|
||||
|
||||
template <typename...>
|
||||
inline constexpr bool no_duplicate_options = true;
|
||||
|
||||
template <typename Option, typename... Rest>
|
||||
inline constexpr bool no_duplicate_options<Option, Rest...> =
|
||||
!__option_present_in_list<Option, Rest...> && no_duplicate_options<Rest...>;
|
||||
} // namespace __detail
|
||||
|
||||
/**
|
||||
* @brief Launch option enabling cooperative launch
|
||||
*
|
||||
* This launch option causes the launched grid to be restricted to a number of
|
||||
* blocks that can simultaneously execute on the device. It means that every
|
||||
* thread in the launched grid can eventually observe execution of each other
|
||||
* thread in the grid. It also enables usage of
|
||||
* cooperative_groups::grid_group::sync() function, that synchronizes all
|
||||
* threads in the grid.
|
||||
*
|
||||
* @par Snippet
|
||||
* @code
|
||||
* #include <cuda/launch>
|
||||
* #include <cooperative_groups.h>
|
||||
*
|
||||
* template <typename Configuration>
|
||||
* __global__ void kernel(Configuration conf)
|
||||
* {
|
||||
* auto grid = cooperative_groups::this_grid();
|
||||
* grid.sync();
|
||||
* }
|
||||
*
|
||||
* void kernel_launch(cuda::stream_ref stream) {
|
||||
* auto dims = cuda::make_hierarchy(cuda::block<128>(), cuda::grid(4));
|
||||
* auto conf = cuda::make_configuration(dims, cooperative_launch());
|
||||
*
|
||||
* cuda::launch(stream, conf, kernel);
|
||||
* }
|
||||
* @endcode
|
||||
*/
|
||||
struct cooperative_launch : public __detail::launch_option
|
||||
{
|
||||
static constexpr bool needs_attribute_space = true;
|
||||
static constexpr bool is_relevant_on_device = true;
|
||||
static constexpr __detail::launch_option_kind kind = __detail::launch_option_kind::cooperative_launch;
|
||||
};
|
||||
|
||||
[[nodiscard]] _CCCL_API inline cudaError_t
|
||||
__apply_launch_option(const cooperative_launch&, CUlaunchConfig& config, CUfunction) noexcept
|
||||
{
|
||||
CUlaunchAttribute attr;
|
||||
attr.id = CU_LAUNCH_ATTRIBUTE_COOPERATIVE;
|
||||
attr.value.cooperative = true;
|
||||
|
||||
config.attrs[config.numAttrs++] = attr;
|
||||
|
||||
return cudaSuccess;
|
||||
}
|
||||
|
||||
template <class _Tp>
|
||||
class __dyn_smem_option_base
|
||||
{
|
||||
protected:
|
||||
using value_type = _Tp;
|
||||
using view_type = _Tp&;
|
||||
};
|
||||
|
||||
template <class _Tp>
|
||||
class __dyn_smem_option_base<_Tp[]>
|
||||
{
|
||||
protected:
|
||||
using value_type = _Tp;
|
||||
using view_type = ::cuda::std::span<_Tp>;
|
||||
|
||||
::cuda::std::size_t __n_;
|
||||
|
||||
_CCCL_HOST_API constexpr __dyn_smem_option_base(::cuda::std::size_t __n) noexcept
|
||||
: __n_{__n}
|
||||
{}
|
||||
};
|
||||
|
||||
template <class _Tp, ::cuda::std::size_t _Np>
|
||||
class __dyn_smem_option_base<_Tp[_Np]>
|
||||
{
|
||||
protected:
|
||||
using value_type = _Tp;
|
||||
using view_type = ::cuda::std::span<_Tp, _Np>;
|
||||
|
||||
static constexpr ::cuda::std::size_t __n_ = _Np;
|
||||
};
|
||||
|
||||
enum class non_portable_t : unsigned char
|
||||
{
|
||||
};
|
||||
inline constexpr non_portable_t non_portable{};
|
||||
|
||||
inline constexpr ::cuda::std::size_t __max_portable_dyn_smem_size = 48 * 1024;
|
||||
|
||||
/**
|
||||
* @brief Launch option specifying dynamic shared memory configuration
|
||||
*
|
||||
* This launch option causes the launch to allocate amount of shared memory
|
||||
* sufficient to store the specified number of object of the specified type.
|
||||
* This type can be constructed with dynamic_shared_memory helper function.
|
||||
*
|
||||
* When launch configuration contains this option, that configuration can be
|
||||
* then passed to dynamic_shared_memory to get the view_type over the
|
||||
* dynamic shared memory. It is also possible to obtain that memory through
|
||||
* the original extern __shared__ variable[] declaration.
|
||||
*
|
||||
* CUDA guarantees that each device has at least 48kB of shared memory
|
||||
* per block, but most devices have more than that.
|
||||
* In order to allocate more dynamic shared memory than the portable
|
||||
* limit, opt-in NonPortableSize template argument should be set to true,
|
||||
* otherwise kernel launch will fail.
|
||||
*
|
||||
* @par Snippet
|
||||
* @code
|
||||
* #include <cuda/launch>
|
||||
*
|
||||
* template <typename Configuration>
|
||||
* __global__ void kernel(Configuration conf)
|
||||
* {
|
||||
* auto dynamic_shared = cuda::dynamic_shared_memory(conf);
|
||||
* dynamic_shared[0] = 1;
|
||||
* }
|
||||
*
|
||||
* void kernel_launch(cuda::stream_ref stream) {
|
||||
* auto dims = cuda::make_hierarchy(cuda::block<128>(), cuda::grid(4));
|
||||
* auto conf = cuda::make_configuration(dims,
|
||||
* cuda::dynamic_shared_memory<int[128]>());
|
||||
*
|
||||
* cuda::launch(stream, conf, kernel);
|
||||
* }
|
||||
* @endcode
|
||||
* @par
|
||||
*
|
||||
* @tparam Content
|
||||
* Type intended to be stored in dynamic shared memory
|
||||
*
|
||||
* @tparam Extent
|
||||
* Statically specified number of Content objects in dynamic shared memory,
|
||||
* or cuda::std::dynamic_extent, if its dynamic
|
||||
*
|
||||
* @tparam NonPortableSize
|
||||
* Needs to be enabled to exceed the portable limit of 48kB of shared memory
|
||||
* per block
|
||||
*/
|
||||
template <class _Tp>
|
||||
class _CCCL_DECLSPEC_EMPTY_BASES dynamic_shared_memory_option
|
||||
: __dyn_smem_option_base<_Tp>
|
||||
, public __detail::launch_option
|
||||
{
|
||||
using __base_type = __dyn_smem_option_base<_Tp>;
|
||||
|
||||
static_assert(::cuda::std::rank_v<_Tp> <= 1,
|
||||
"multidimensional arrays cannot be used with dynamic shared "
|
||||
"memory option");
|
||||
static_assert(!::cuda::std::is_const_v<typename __base_type::value_type>, "the value type cannot be const");
|
||||
static_assert(!::cuda::std::is_reference_v<typename __base_type::value_type>, "the value type cannot be a reference");
|
||||
|
||||
public:
|
||||
bool __non_portable_{}; //!< \c true if the object was created with
|
||||
//!< non_portable flag.
|
||||
|
||||
using typename __base_type::value_type; //!< Value type of the dynamic
|
||||
//!< shared memory elements.
|
||||
using typename __base_type::view_type; //!< The view type returned by the
|
||||
//!< cuda::dynamic_shared_memory(config).
|
||||
|
||||
static constexpr bool is_relevant_on_device = true;
|
||||
static constexpr __detail::launch_option_kind kind = __detail::launch_option_kind::dynamic_shared_memory;
|
||||
|
||||
//! @brief Gets the size of the dynamic shared memory in bytes.
|
||||
[[nodiscard]] _CCCL_API constexpr ::cuda::std::size_t size_bytes() const noexcept
|
||||
{
|
||||
if constexpr (::cuda::std::is_unbounded_array_v<_Tp>)
|
||||
{
|
||||
# if !_CCCL_TILE_COMPILATION() // error: asm statement is unsupported in tile code
|
||||
_CCCL_IF_NOT_CONSTEVAL_DEFAULT
|
||||
{
|
||||
NV_IF_TARGET(NV_IS_DEVICE, (return ::cuda::ptx::get_sreg_dynamic_smem_size();))
|
||||
}
|
||||
# endif // !_CCCL_TILE_COMPILATION()
|
||||
return __base_type::__n_ * sizeof(value_type);
|
||||
}
|
||||
else
|
||||
{
|
||||
return sizeof(_Tp);
|
||||
}
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr view_type __make_view(value_type* __ptr) const noexcept
|
||||
{
|
||||
if constexpr (::cuda::std::rank_v<_Tp> == 0)
|
||||
{
|
||||
return *__ptr;
|
||||
}
|
||||
else
|
||||
{
|
||||
return view_type{__ptr, __base_type::__n_};
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to access private constructors
|
||||
static constexpr dynamic_shared_memory_option __create(bool __non_portable = false) noexcept
|
||||
{
|
||||
return dynamic_shared_memory_option{__non_portable};
|
||||
}
|
||||
|
||||
static constexpr dynamic_shared_memory_option __create(::cuda::std::size_t __n, bool __non_portable = false) noexcept
|
||||
{
|
||||
return dynamic_shared_memory_option{__n, __non_portable};
|
||||
}
|
||||
|
||||
private:
|
||||
_CCCL_HOST_API constexpr dynamic_shared_memory_option(bool __non_portable = false) noexcept
|
||||
: __non_portable_{__non_portable}
|
||||
{}
|
||||
|
||||
_CCCL_HOST_API constexpr dynamic_shared_memory_option(::cuda::std::size_t __n, bool __non_portable = false) noexcept
|
||||
: __base_type{__n}
|
||||
, __non_portable_{__non_portable}
|
||||
{}
|
||||
};
|
||||
|
||||
template <class _Tp>
|
||||
[[nodiscard]] ::cudaError_t __apply_launch_option(
|
||||
const dynamic_shared_memory_option<_Tp>& __opt, ::CUlaunchConfig& __config, ::CUfunction __kernel) noexcept
|
||||
{
|
||||
::cudaError_t __status = ::cudaSuccess;
|
||||
|
||||
// Since CUDA 12.4, querying CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES requires
|
||||
// the function to be loaded.
|
||||
if (::cuda::__driver::__version_at_least(12, 4))
|
||||
{
|
||||
__status = ::cuda::__driver::__functionLoadNoThrow(__kernel);
|
||||
if (__status != ::cudaSuccess)
|
||||
{
|
||||
return __status;
|
||||
}
|
||||
}
|
||||
|
||||
int __static_smem_size{};
|
||||
__status = ::cuda::__driver::__functionGetAttributeNoThrow(
|
||||
__static_smem_size, ::CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, __kernel);
|
||||
if (__status != ::cudaSuccess)
|
||||
{
|
||||
return __status;
|
||||
}
|
||||
|
||||
int __max_dyn_smem_size{};
|
||||
__status = ::cuda::__driver::__functionGetAttributeNoThrow(
|
||||
__max_dyn_smem_size, ::CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, __kernel);
|
||||
if (__status != ::cudaSuccess)
|
||||
{
|
||||
return __status;
|
||||
}
|
||||
|
||||
const auto __dyn_smem_size = ::cuda::overflow_cast<int>(__opt.size_bytes());
|
||||
if (__dyn_smem_size.overflow)
|
||||
{
|
||||
return ::cudaErrorInvalidValue;
|
||||
}
|
||||
|
||||
const int __smem_size = __static_smem_size + __dyn_smem_size.value;
|
||||
if (static_cast<::cuda::std::size_t>(__smem_size) > __max_portable_dyn_smem_size && !__opt.__non_portable_)
|
||||
{
|
||||
return ::cudaErrorInvalidValue;
|
||||
}
|
||||
|
||||
if (__max_dyn_smem_size < __dyn_smem_size.value)
|
||||
{
|
||||
__status = ::cuda::__driver::__functionSetAttributeNoThrow(
|
||||
__kernel, ::CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, __dyn_smem_size.value);
|
||||
if (__status != ::cudaSuccess)
|
||||
{
|
||||
return __status;
|
||||
}
|
||||
}
|
||||
|
||||
__config.sharedMemBytes = static_cast<unsigned>(__dyn_smem_size.value);
|
||||
return ::cudaSuccess;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Function that creates dynamic_shared_memory_option for non-unbounded array types
|
||||
*
|
||||
* @tparam _Tp Type intended to be stored in dynamic shared memory (must not be an unbounded array)
|
||||
* @return dynamic_shared_memory_option<_Tp> instance
|
||||
*/
|
||||
_CCCL_TEMPLATE(class _Tp)
|
||||
_CCCL_REQUIRES((!::cuda::std::is_unbounded_array_v<_Tp>) )
|
||||
[[nodiscard]] _CCCL_HOST_API constexpr dynamic_shared_memory_option<_Tp> dynamic_shared_memory() noexcept
|
||||
{
|
||||
static_assert(sizeof(_Tp) <= __max_portable_dyn_smem_size, "portable dynamic shared memory limit exceeded");
|
||||
return dynamic_shared_memory_option<_Tp>::__create(false);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Function that creates dynamic_shared_memory_option for non-unbounded array types with non-portable flag
|
||||
*
|
||||
* @tparam _Tp Type intended to be stored in dynamic shared memory (must not be an unbounded array)
|
||||
* @note Pass cuda::non_portable to opt in to non-portable shared memory sizes.
|
||||
* @return dynamic_shared_memory_option<_Tp> instance
|
||||
*/
|
||||
_CCCL_TEMPLATE(class _Tp)
|
||||
_CCCL_REQUIRES((!::cuda::std::is_unbounded_array_v<_Tp>) )
|
||||
[[nodiscard]] _CCCL_HOST_API constexpr dynamic_shared_memory_option<_Tp> dynamic_shared_memory(non_portable_t) noexcept
|
||||
{
|
||||
return dynamic_shared_memory_option<_Tp>::__create(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Function that creates dynamic_shared_memory_option for unbounded array types
|
||||
*
|
||||
* @tparam _Tp Unbounded array type
|
||||
* @param __n Number of elements in the dynamic shared memory
|
||||
* @return dynamic_shared_memory_option<_Tp> instance
|
||||
*/
|
||||
_CCCL_TEMPLATE(class _Tp)
|
||||
_CCCL_REQUIRES(::cuda::std::is_unbounded_array_v<_Tp>)
|
||||
[[nodiscard]] _CCCL_HOST_API constexpr dynamic_shared_memory_option<_Tp> dynamic_shared_memory(::cuda::std::size_t __n)
|
||||
{
|
||||
using value_type = typename dynamic_shared_memory_option<_Tp>::value_type;
|
||||
if (__n * sizeof(value_type) > __max_portable_dyn_smem_size)
|
||||
{
|
||||
_CCCL_THROW(::std::invalid_argument, "portable dynamic shared memory limit exceeded");
|
||||
}
|
||||
return dynamic_shared_memory_option<_Tp>::__create(__n, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Function that creates dynamic_shared_memory_option for unbounded array types with non-portable flag
|
||||
*
|
||||
* @tparam _Tp Unbounded array type
|
||||
* @param __n Number of elements in the dynamic shared memory
|
||||
* @note Pass cuda::non_portable to opt in to non-portable shared memory sizes.
|
||||
* @return dynamic_shared_memory_option<_Tp> instance
|
||||
*/
|
||||
_CCCL_TEMPLATE(class _Tp)
|
||||
_CCCL_REQUIRES(::cuda::std::is_unbounded_array_v<_Tp>)
|
||||
[[nodiscard]] _CCCL_HOST_API constexpr dynamic_shared_memory_option<_Tp>
|
||||
dynamic_shared_memory(::cuda::std::size_t __n, non_portable_t) noexcept
|
||||
{
|
||||
return dynamic_shared_memory_option<_Tp>::__create(__n, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Launch option specifying launch priority
|
||||
*
|
||||
* This launch option causes the launched grid to be scheduled with the
|
||||
* specified priority. More about stream priorities and valid values can be
|
||||
* found in the CUDA programming guide `here
|
||||
* <https://docs.nvidia.com/cuda/cuda-c-programming-guide/#stream-priorities>`_
|
||||
*/
|
||||
struct launch_priority : public __detail::launch_option
|
||||
{
|
||||
static constexpr bool needs_attribute_space = true;
|
||||
static constexpr bool is_relevant_on_device = false;
|
||||
static constexpr __detail::launch_option_kind kind = __detail::launch_option_kind::launch_priority;
|
||||
int priority;
|
||||
|
||||
launch_priority(int p) noexcept
|
||||
: priority(p)
|
||||
{}
|
||||
};
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_API inline cudaError_t
|
||||
__apply_launch_option(const launch_priority& __opt, CUlaunchConfig& config, CUfunction) noexcept
|
||||
{
|
||||
CUlaunchAttribute attr;
|
||||
attr.id = CU_LAUNCH_ATTRIBUTE_PRIORITY;
|
||||
attr.value.priority = __opt.priority;
|
||||
|
||||
config.attrs[config.numAttrs++] = attr;
|
||||
|
||||
return cudaSuccess;
|
||||
}
|
||||
|
||||
template <typename... _OptionsToFilter>
|
||||
struct __filter_options
|
||||
{
|
||||
template <bool _Pred, typename _Option>
|
||||
[[nodiscard]] auto __option_or_empty(const _Option& __option)
|
||||
{
|
||||
if constexpr (_Pred)
|
||||
{
|
||||
return ::cuda::std::tuple(__option);
|
||||
}
|
||||
else
|
||||
{
|
||||
return ::cuda::std::tuple{};
|
||||
}
|
||||
}
|
||||
|
||||
template <typename... _Options>
|
||||
[[nodiscard]] auto operator()(const _Options&... __options)
|
||||
{
|
||||
return ::cuda::std::tuple_cat(
|
||||
__option_or_empty<!__detail::__option_present_in_list<_Options, _OptionsToFilter...>>(__options)...);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename _Dimensions, typename... _Options>
|
||||
auto __make_config_from_tuple(const _Dimensions& __dims, const ::cuda::std::tuple<_Options...>& __opts);
|
||||
|
||||
template <typename _Tp>
|
||||
inline constexpr bool __is_kernel_config = false;
|
||||
|
||||
template <typename _Dimensions, typename... _Options>
|
||||
inline constexpr bool __is_kernel_config<kernel_config<_Dimensions, _Options...>> = true;
|
||||
|
||||
template <typename _Tp>
|
||||
_CCCL_CONCEPT __kernel_has_default_config =
|
||||
_CCCL_REQUIRES_EXPR((_Tp), _Tp& __t)(requires(__is_kernel_config<decltype(__t.default_config())>));
|
||||
|
||||
/**
|
||||
* @brief Type describing a kernel launch configuration
|
||||
*
|
||||
* This type should not be constructed directly and make_config helper
|
||||
* function should be used instead
|
||||
*
|
||||
* @tparam Dimensions
|
||||
* cuda::hierarchy instance that describes dimensions
|
||||
* of thread hierarchy in this configuration object
|
||||
*
|
||||
* @tparam Options
|
||||
* Types of options that were added to this configuration object
|
||||
*/
|
||||
template <typename Hierarchy, typename... Options>
|
||||
struct kernel_config
|
||||
{
|
||||
using hierarchy_type = Hierarchy;
|
||||
using options_type = ::cuda::std::tuple<Options...>;
|
||||
|
||||
static_assert(::cuda::std::_And<::cuda::std::is_base_of<__detail::launch_option, Options>...>::value);
|
||||
static_assert(__detail::no_duplicate_options<Options...>);
|
||||
|
||||
constexpr kernel_config(const Hierarchy& hierarchy, const Options&... opts)
|
||||
: __hierarchy(hierarchy)
|
||||
, __options(opts...) {};
|
||||
constexpr kernel_config(const Hierarchy& hierarchy, const ::cuda::std::tuple<Options...>& opts)
|
||||
: __hierarchy(hierarchy)
|
||||
, __options(opts) {};
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr const Hierarchy& hierarchy() const noexcept
|
||||
{
|
||||
return __hierarchy;
|
||||
}
|
||||
|
||||
[[nodiscard]] _CCCL_API constexpr const ::cuda::std::tuple<Options...>& options() const noexcept
|
||||
{
|
||||
return __options;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Add a new option to this configuration
|
||||
*
|
||||
* Returns a new kernel_config that has all option and dimensions from this
|
||||
* kernel_config with the option from the argument added to it
|
||||
*
|
||||
* @param new_options
|
||||
* Options to be added to the configuration
|
||||
*/
|
||||
template <typename... NewOptions>
|
||||
[[nodiscard]] auto add(const NewOptions&... new_options) const
|
||||
{
|
||||
return kernel_config<Hierarchy, Options..., NewOptions...>(
|
||||
__hierarchy, ::cuda::std::tuple_cat(__options, ::cuda::std::make_tuple(new_options...)));
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Combine this configuration with another configuration object
|
||||
*
|
||||
* Returns a new `kernel_config` that is a combination of this configuration
|
||||
* and the configuration from argument. It contains dimensions that are
|
||||
* combination of dimensions in this object and the other configuration. The
|
||||
* resulting hierarchy holds levels present in both hierarchies. In case of
|
||||
* overlap of levels hierarchy from this configuration is prioritized, so
|
||||
* the result always holds all levels from this hierarchy and
|
||||
* non-overlapping levels from the other hierarchy. This behavior is the
|
||||
* same as `combine()` member function of the hierarchy type. The result
|
||||
* also contains configuration options from both configurations. In case the
|
||||
* same type of a configuration option is present in both configuration this
|
||||
* configuration is copied into the resulting configuration.
|
||||
*
|
||||
* @param __other_config
|
||||
* Other configuration to combine with this configuration
|
||||
*/
|
||||
template <typename _OtherDimensions, typename... _OtherOptions>
|
||||
[[nodiscard]] auto combine(const kernel_config<_OtherDimensions, _OtherOptions...>& __other_config) const
|
||||
{
|
||||
// can't use fully qualified kernel_config name here because of nvcc bug,
|
||||
// TODO remove __make_config_from_tuple once fixed
|
||||
return __make_config_from_tuple(
|
||||
__hierarchy.combine(__other_config.hierarchy()),
|
||||
::cuda::std::tuple_cat(__options, ::cuda::std::apply(__filter_options<Options...>{}, __other_config.options())));
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Combine this configuration with default configuration of a kernel
|
||||
* functor
|
||||
*
|
||||
* Returns a new `kernel_config` that is a combination of this configuration
|
||||
* and a default configuration from the kernel argument. Default
|
||||
* configuration is a `kernel_config` object returned from
|
||||
* `default_config()` member function of the kernel type. The configurations
|
||||
* are combined using the `combine()` member function of this configuration.
|
||||
* If the kernel has no default configuration, a copy of this configuration
|
||||
* is returned without any changes.
|
||||
*
|
||||
* @param __kernel
|
||||
* Kernel functor to search for the default configuration
|
||||
*/
|
||||
template <typename _Kernel>
|
||||
[[nodiscard]] auto combine_with_default(const _Kernel& __kernel) const
|
||||
{
|
||||
if constexpr (__kernel_has_default_config<_Kernel>)
|
||||
{
|
||||
return combine(__kernel.default_config());
|
||||
}
|
||||
else
|
||||
{
|
||||
return *this;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
Hierarchy __hierarchy;
|
||||
::cuda::std::tuple<Options...> __options;
|
||||
};
|
||||
|
||||
// We can consider removing the operator&, but its convenient for in-line
|
||||
// construction
|
||||
template <typename Dimensions, typename... Options, typename NewLevel>
|
||||
_CCCL_HOST_API constexpr auto
|
||||
operator&(const kernel_config<Dimensions, Options...>& config, const NewLevel& new_level) noexcept
|
||||
{
|
||||
return kernel_config(hierarchy_add_level(config.hierarchy(), new_level), config.options());
|
||||
}
|
||||
|
||||
template <typename NewLevel, typename Dimensions, typename... Options>
|
||||
_CCCL_HOST_API constexpr auto
|
||||
operator&(const NewLevel& new_level, const kernel_config<Dimensions, Options...>& config) noexcept
|
||||
{
|
||||
return kernel_config(hierarchy_add_level(config.hierarchy(), new_level), config.options());
|
||||
}
|
||||
|
||||
template <typename L1, typename Dims1, typename L2, typename Dims2>
|
||||
_CCCL_HOST_API constexpr auto
|
||||
operator&(const hierarchy_level_desc<L1, Dims1>& l1, const hierarchy_level_desc<L2, Dims2>& l2) noexcept
|
||||
{
|
||||
return kernel_config(::cuda::make_hierarchy(l1, l2));
|
||||
}
|
||||
|
||||
template <typename _Dimensions, typename... _Options>
|
||||
auto __make_config_from_tuple(const _Dimensions& __dims, const ::cuda::std::tuple<_Options...>& __opts)
|
||||
{
|
||||
return kernel_config(__dims, __opts);
|
||||
}
|
||||
|
||||
template <typename Dimensions,
|
||||
typename... Options,
|
||||
typename Option,
|
||||
typename = ::cuda::std::enable_if_t<::cuda::std::is_base_of_v<__detail::launch_option, Option>>>
|
||||
[[nodiscard]] constexpr auto
|
||||
operator&(const kernel_config<Dimensions, Options...>& config, const Option& option) noexcept
|
||||
{
|
||||
return config.add(option);
|
||||
}
|
||||
|
||||
template <typename... Levels,
|
||||
typename Option,
|
||||
typename = ::cuda::std::enable_if_t<::cuda::std::is_base_of_v<__detail::launch_option, Option>>>
|
||||
[[nodiscard]] constexpr auto operator&(const hierarchy<Levels...>& dims, const Option& option) noexcept
|
||||
{
|
||||
return kernel_config(dims, option);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Construct kernel configuration
|
||||
*
|
||||
* This function takes thread hierarchy dimensions description and any number of
|
||||
* launch options and combines them into kernel configuration object. It can be
|
||||
* then used along with kernel function and its argument to launch that kernel
|
||||
* with the specified dimensions and options
|
||||
*
|
||||
* @param dims
|
||||
* Object describing dimensions of the thread hierarchy in the resulting kernel
|
||||
* configuration object
|
||||
*
|
||||
* @param opts
|
||||
* Variadic number of launch configuration options to be included in the
|
||||
* resulting kernel configuration object
|
||||
*/
|
||||
template <typename BottomUnit, typename... Levels, typename... Opts>
|
||||
[[nodiscard]] constexpr auto make_config(const hierarchy<BottomUnit, Levels...>& dims, const Opts&... opts) noexcept
|
||||
{
|
||||
return kernel_config<hierarchy<BottomUnit, Levels...>, Opts...>(dims, opts...);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief A shorthand for creating a kernel configuration with a hierarchy of
|
||||
* CUDA threads evenly distributing elements among blocks and threads.
|
||||
*
|
||||
* @par Snippet
|
||||
* @code
|
||||
* #include <cuda/hierarchy_dimensions.cuh>
|
||||
* using namespace cuda;
|
||||
*
|
||||
* constexpr int threadsPerBlock = 256;
|
||||
* auto dims = distribute<threadsPerBlock>(numElements);
|
||||
*
|
||||
* // Equivalent to:
|
||||
* constexpr int threadsPerBlock = 256;
|
||||
* int blocksPerGrid = (numElements + threadsPerBlock - 1) / threadsPerBlock;
|
||||
* auto dims = make_hierarchy(grid_dims(blocksPerGrid),
|
||||
* block_dims<threadsPerBlock>());
|
||||
* @endcode
|
||||
*/
|
||||
template <int _ThreadsPerBlock>
|
||||
constexpr auto distribute(int numElements) noexcept
|
||||
{
|
||||
int blocksPerGrid = (numElements + _ThreadsPerBlock - 1) / _ThreadsPerBlock;
|
||||
return make_config(make_hierarchy(grid_dims(blocksPerGrid), block_dims<_ThreadsPerBlock>()));
|
||||
}
|
||||
|
||||
template <typename... Prev>
|
||||
[[nodiscard]] constexpr auto __process_config_args(const ::cuda::std::tuple<Prev...>& previous)
|
||||
{
|
||||
if constexpr (sizeof...(Prev) == 0)
|
||||
{
|
||||
return kernel_config<__empty_hierarchy>(__empty_hierarchy());
|
||||
}
|
||||
else
|
||||
{
|
||||
constexpr auto fn = &make_hierarchy<void, const Prev&...>;
|
||||
return kernel_config(::cuda::std::apply(fn, previous));
|
||||
}
|
||||
}
|
||||
|
||||
template <typename... Prev, typename Arg, typename... Rest>
|
||||
[[nodiscard]] constexpr auto
|
||||
__process_config_args(const ::cuda::std::tuple<Prev...>& previous, const Arg& arg, const Rest&... rest)
|
||||
{
|
||||
if constexpr (::cuda::std::is_base_of_v<__detail::launch_option, Arg>)
|
||||
{
|
||||
static_assert((::cuda::std::is_base_of_v<__detail::launch_option, Rest> && ...),
|
||||
"Hierarchy levels and launch options can't be mixed");
|
||||
if constexpr (sizeof...(Prev) == 0)
|
||||
{
|
||||
return kernel_config(__empty_hierarchy(), arg, rest...);
|
||||
}
|
||||
else
|
||||
{
|
||||
constexpr auto fn = make_hierarchy<void, const Prev&...>;
|
||||
return kernel_config(::cuda::std::apply(fn, previous), arg, rest...);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
return __process_config_args(::cuda::std::tuple_cat(previous, ::cuda::std::make_tuple(arg)), rest...);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
[[nodiscard]] constexpr auto make_config(const Args&... args)
|
||||
{
|
||||
return __process_config_args(::cuda::std::make_tuple(), args...);
|
||||
}
|
||||
|
||||
namespace __detail
|
||||
{
|
||||
template <typename Dimensions, typename... Options>
|
||||
inline unsigned int constexpr kernel_config_count_attr_space(const kernel_config<Dimensions, Options...>&) noexcept
|
||||
{
|
||||
return (0 + ... + Options::needs_attribute_space);
|
||||
}
|
||||
|
||||
template <typename Dimensions, typename... Options>
|
||||
[[nodiscard]] cudaError_t apply_kernel_config(
|
||||
const kernel_config<Dimensions, Options...>& config, CUlaunchConfig& cuda_config, CUfunction kernel) noexcept
|
||||
{
|
||||
return ::cuda::std::apply(
|
||||
[&](auto&... config_options) {
|
||||
cudaError_t __status = cudaSuccess;
|
||||
|
||||
// Use short-cutting && to skip the rest on error, is this too
|
||||
// convoluted?
|
||||
// For some reason gcc 7 complains about __status capture, so we pass it as a reference
|
||||
(void) (... && [](cudaError_t call_status, cudaError_t& __status_out) {
|
||||
__status_out = call_status;
|
||||
return call_status == cudaSuccess;
|
||||
}(::cuda::__apply_launch_option(config_options, cuda_config, kernel), __status));
|
||||
|
||||
return __status;
|
||||
},
|
||||
config.options());
|
||||
}
|
||||
} // namespace __detail
|
||||
|
||||
# if _CCCL_CUDA_COMPILATION()
|
||||
|
||||
template <class _Dims, class... _Opts>
|
||||
_CCCL_DEVICE_API decltype(auto) dynamic_shared_memory(const kernel_config<_Dims, _Opts...>& __config) noexcept
|
||||
{
|
||||
auto& __opt = __detail::find_option_in_tuple<__detail::launch_option_kind::dynamic_shared_memory>(__config.options());
|
||||
using _Opt = ::cuda::std::remove_reference_t<decltype(__opt)>;
|
||||
static_assert(!::cuda::std::is_same_v<_Opt, __detail::option_not_found>,
|
||||
"Dynamic shared memory option not found in the kernel configuration");
|
||||
extern __shared__ unsigned char __cccl_device_dyn_smem[];
|
||||
return __opt.__make_view(reinterpret_cast<typename _Opt::value_type*>(__cccl_device_dyn_smem));
|
||||
}
|
||||
|
||||
# endif // _CCCL_CUDA_COMPILATION()
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
#endif // _CUDA___LAUNCH_CONFIGURATION_H
|
||||
119
cccl_upstream/libcudacxx/include/cuda/__launch/host_launch.h
Normal file
119
cccl_upstream/libcudacxx/include/cuda/__launch/host_launch.h
Normal file
@@ -0,0 +1,119 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA___LAUNCH_HOST_LAUNCH_H
|
||||
#define _CUDA___LAUNCH_HOST_LAUNCH_H
|
||||
|
||||
#include <cuda/__cccl_config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
# include <cuda/__driver/driver_api.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__functional/reference_wrapper.h>
|
||||
# include <cuda/std/__memory/addressof.h>
|
||||
# include <cuda/std/__tuple_dir/apply.h>
|
||||
# include <cuda/std/__tuple_dir/tuple.h>
|
||||
# include <cuda/std/__type_traits/decay.h>
|
||||
# include <cuda/std/__type_traits/is_function.h>
|
||||
# include <cuda/std/__type_traits/is_move_constructible.h>
|
||||
# include <cuda/std/__type_traits/is_pointer.h>
|
||||
# include <cuda/std/__utility/move.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA
|
||||
|
||||
template <class _Callable>
|
||||
_CCCL_HOST_API inline void CUDA_CB __host_func_launcher(void* __callable_ptr)
|
||||
{
|
||||
(*(_Callable*) __callable_ptr)();
|
||||
}
|
||||
|
||||
template <class _Callable, class... _Args>
|
||||
struct __stream_callback_data
|
||||
{
|
||||
_Callable __callable_;
|
||||
::cuda::std::tuple<_Args...> __args_;
|
||||
};
|
||||
|
||||
template <class _CallbackData>
|
||||
_CCCL_HOST_API inline void CUDA_CB __stream_callback_launcher(::CUstream, ::CUresult __status, void* __data_ptr)
|
||||
{
|
||||
auto* __casted_data_ptr = static_cast<_CallbackData*>(__data_ptr);
|
||||
if (__status == ::CUDA_SUCCESS)
|
||||
{
|
||||
(void) ::cuda::std::apply(__casted_data_ptr->__callable_, ::cuda::std::move(__casted_data_ptr->__args_));
|
||||
}
|
||||
delete __casted_data_ptr;
|
||||
}
|
||||
|
||||
//! @brief Launches a host callable to be executed in stream order on the provided stream
|
||||
//!
|
||||
//! Callable and arguments are copied into an internal dynamic allocation to preserve them
|
||||
//! until the asynchronous call happens. Lambda capture or reference_wrapper can be used if
|
||||
//! there is a need to pass something by reference.
|
||||
//!
|
||||
//! Callable must not call any APIs from cuda, thrust or cub namespaces.
|
||||
//! It must not call into CUDA Runtime or Driver APIs. It also can't depend on another
|
||||
//! thread that might block on any asynchronous CUDA work.
|
||||
//!
|
||||
//! @param __stream Stream to launch the host function on
|
||||
//! @param __callable Host function or callable object to call in stream order
|
||||
//! @param __args Arguments to call the supplied callable with
|
||||
template <class _Callable, class... _Args>
|
||||
_CCCL_HOST_API void host_launch(stream_ref __stream, _Callable __callable, _Args... __args)
|
||||
{
|
||||
static_assert(::cuda::std::is_invocable_v<_Callable, _Args...>,
|
||||
"Callable can't be called with the supplied arguments");
|
||||
static_assert(::cuda::std::is_move_constructible_v<_Callable>, "The callable must be move constructible");
|
||||
static_assert((::cuda::std::is_move_constructible_v<_Args> && ...),
|
||||
"All callback arguments must be move constructible");
|
||||
|
||||
constexpr auto __has_args = sizeof...(_Args) > 0;
|
||||
|
||||
if constexpr (!__has_args && ::cuda::std::is_function_v<_Callable> && ::cuda::std::is_pointer_v<_Callable>)
|
||||
{
|
||||
::cuda::__driver::__launchHostFunc(__stream.get(), ::cuda::__host_func_launcher<_Callable>, (void*) __callable);
|
||||
}
|
||||
else if constexpr (!__has_args && ::cuda::std::__is_cuda_std_reference_wrapper_v<_Callable>)
|
||||
{
|
||||
::cuda::__driver::__launchHostFunc(
|
||||
__stream.get(),
|
||||
::cuda::__host_func_launcher<typename _Callable::type>,
|
||||
(void*) ::cuda::std::addressof(__callable.get()));
|
||||
}
|
||||
else
|
||||
{
|
||||
using _CallbackData = __stream_callback_data<_Callable, _Args...>;
|
||||
_CallbackData* __callback_data_ptr =
|
||||
new _CallbackData{::cuda::std::move(__callable), {::cuda::std::move(__args)...}};
|
||||
|
||||
// We use the callback here to have it execute even on stream error, because it needs to free the above allocation
|
||||
::cuda::__driver::__streamAddCallback(
|
||||
__stream.get(), ::cuda::__stream_callback_launcher<_CallbackData>, __callback_data_ptr);
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
#endif // !_CUDA___LAUNCH_HOST_LAUNCH_H
|
||||
628
cccl_upstream/libcudacxx/include/cuda/__launch/launch.h
Normal file
628
cccl_upstream/libcudacxx/include/cuda/__launch/launch.h
Normal file
@@ -0,0 +1,628 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA___LAUNCH_LAUNCH_H
|
||||
#define _CUDA___LAUNCH_LAUNCH_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
# include <cuda/__driver/driver_api.h>
|
||||
# include <cuda/__hierarchy/hierarchy_levels.h>
|
||||
# include <cuda/__hierarchy/traits.h>
|
||||
# include <cuda/__launch/configuration.h>
|
||||
# include <cuda/__runtime/api_wrapper.h>
|
||||
# include <cuda/__runtime/ensure_current_context.h>
|
||||
# include <cuda/__stream/launch_transform.h>
|
||||
# include <cuda/__stream/stream_ref.h>
|
||||
# include <cuda/std/__exception/cuda_error.h>
|
||||
# include <cuda/std/__exception/exception_macros.h>
|
||||
# include <cuda/std/__type_traits/is_function.h>
|
||||
# include <cuda/std/__type_traits/is_pointer.h>
|
||||
# include <cuda/std/__type_traits/type_identity.h>
|
||||
# include <cuda/std/__utility/forward.h>
|
||||
# include <cuda/std/__utility/pod_tuple.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA
|
||||
|
||||
# if _CCCL_CUDA_COMPILATION()
|
||||
|
||||
// clang-cuda replaces the variables with direct nvvm sreg calls, so we want to assume their return values directly.
|
||||
# if _CCCL_CUDA_COMPILER(CLANG)
|
||||
# define _CCCL_THREAD_IDX_X ::__nvvm_read_ptx_sreg_tid_x()
|
||||
# define _CCCL_THREAD_IDX_Y ::__nvvm_read_ptx_sreg_tid_y()
|
||||
# define _CCCL_THREAD_IDX_Z ::__nvvm_read_ptx_sreg_tid_z()
|
||||
# define _CCCL_BLOCK_DIM_X ::__nvvm_read_ptx_sreg_ntid_x()
|
||||
# define _CCCL_BLOCK_DIM_Y ::__nvvm_read_ptx_sreg_ntid_y()
|
||||
# define _CCCL_BLOCK_DIM_Z ::__nvvm_read_ptx_sreg_ntid_z()
|
||||
# define _CCCL_BLOCK_IDX_X ::__nvvm_read_ptx_sreg_ctaid_x()
|
||||
# define _CCCL_BLOCK_IDX_Y ::__nvvm_read_ptx_sreg_ctaid_y()
|
||||
# define _CCCL_BLOCK_IDX_Z ::__nvvm_read_ptx_sreg_ctaid_z()
|
||||
# define _CCCL_CLUSTER_DIM_X ::__nvvm_read_ptx_sreg_cluster_nctaid_x()
|
||||
# define _CCCL_CLUSTER_DIM_Y ::__nvvm_read_ptx_sreg_cluster_nctaid_y()
|
||||
# define _CCCL_CLUSTER_DIM_Z ::__nvvm_read_ptx_sreg_cluster_nctaid_z()
|
||||
# define _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_X ::__nvvm_read_ptx_sreg_cluster_ctaid_x()
|
||||
# define _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Y ::__nvvm_read_ptx_sreg_cluster_ctaid_y()
|
||||
# define _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Z ::__nvvm_read_ptx_sreg_cluster_ctaid_z()
|
||||
# define _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_X ::__nvvm_read_ptx_sreg_nclusterid_x()
|
||||
# define _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Y ::__nvvm_read_ptx_sreg_nclusterid_y()
|
||||
# define _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Z ::__nvvm_read_ptx_sreg_nclusterid_z()
|
||||
# define _CCCL_CLUSTER_IDX_X ::__nvvm_read_ptx_sreg_clusterid_x()
|
||||
# define _CCCL_CLUSTER_IDX_Y ::__nvvm_read_ptx_sreg_clusterid_y()
|
||||
# define _CCCL_CLUSTER_IDX_Z ::__nvvm_read_ptx_sreg_clusterid_z()
|
||||
# define _CCCL_CLUSTER_RELATIVE_BLOCK_RANK ::__nvvm_read_ptx_sreg_cluster_ctarank()
|
||||
# define _CCCL_CLUSTER_SIZE_IN_BLOCKS ::__nvvm_read_ptx_sreg_cluster_nctarank()
|
||||
# define _CCCL_GRID_DIM_X ::__nvvm_read_ptx_sreg_nctaid_x()
|
||||
# define _CCCL_GRID_DIM_Y ::__nvvm_read_ptx_sreg_nctaid_y()
|
||||
# define _CCCL_GRID_DIM_Z ::__nvvm_read_ptx_sreg_nctaid_z()
|
||||
# else // ^^^ _CCCL_CUDA_COMPILER(CLANG) ^^^ / vvv !_CCCL_CUDA_COMPILER(CLANG) vvv
|
||||
# define _CCCL_THREAD_IDX_X threadIdx.x
|
||||
# define _CCCL_THREAD_IDX_Y threadIdx.y
|
||||
# define _CCCL_THREAD_IDX_Z threadIdx.z
|
||||
# define _CCCL_BLOCK_DIM_X blockDim.x
|
||||
# define _CCCL_BLOCK_DIM_Y blockDim.y
|
||||
# define _CCCL_BLOCK_DIM_Z blockDim.z
|
||||
# define _CCCL_BLOCK_IDX_X blockIdx.x
|
||||
# define _CCCL_BLOCK_IDX_Y blockIdx.y
|
||||
# define _CCCL_BLOCK_IDX_Z blockIdx.z
|
||||
# define _CCCL_CLUSTER_DIM_X ::__clusterDim().x
|
||||
# define _CCCL_CLUSTER_DIM_Y ::__clusterDim().y
|
||||
# define _CCCL_CLUSTER_DIM_Z ::__clusterDim().z
|
||||
# define _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_X ::__clusterRelativeBlockIdx().x
|
||||
# define _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Y ::__clusterRelativeBlockIdx().y
|
||||
# define _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Z ::__clusterRelativeBlockIdx().z
|
||||
# define _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_X ::__clusterGridDimInClusters().x
|
||||
# define _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Y ::__clusterGridDimInClusters().y
|
||||
# define _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Z ::__clusterGridDimInClusters().z
|
||||
# define _CCCL_CLUSTER_IDX_X ::__clusterIdx().x
|
||||
# define _CCCL_CLUSTER_IDX_Y ::__clusterIdx().y
|
||||
# define _CCCL_CLUSTER_IDX_Z ::__clusterIdx().z
|
||||
# define _CCCL_CLUSTER_RELATIVE_BLOCK_RANK ::__clusterRelativeBlockRank()
|
||||
# define _CCCL_CLUSTER_SIZE_IN_BLOCKS ::__clusterSizeInBlocks()
|
||||
# define _CCCL_GRID_DIM_X gridDim.x
|
||||
# define _CCCL_GRID_DIM_Y gridDim.y
|
||||
# define _CCCL_GRID_DIM_Z gridDim.z
|
||||
# endif // ^^^ !_CCCL_CUDA_COMPILER(CLANG) ^^^
|
||||
|
||||
// clang-cuda sometimes warns about the assumption being ignored because it contains (potential) side-effects. We can
|
||||
// just suppress it, because in the worst case, the assumption will be just ignored.
|
||||
// note(dabayer): I haven't found out when exactly the assumption fails.
|
||||
_CCCL_DIAG_PUSH
|
||||
_CCCL_DIAG_SUPPRESS_CLANG("-Wassume")
|
||||
|
||||
template <class _Hierarchy>
|
||||
_CCCL_DEVICE_API _CCCL_FORCEINLINE void __assume_known_info() noexcept
|
||||
{
|
||||
static_assert(__is_hierarchy_v<_Hierarchy>);
|
||||
|
||||
constexpr auto __dext = ::cuda::std::dynamic_extent;
|
||||
|
||||
using _BlockDesc = typename _Hierarchy::template level_desc_type<block_level>;
|
||||
using _BlockExts = typename _BlockDesc::extents_type;
|
||||
|
||||
if constexpr (_BlockExts::static_extent(0) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_BLOCK_DIM_X == _BlockExts::static_extent(0));
|
||||
_CCCL_ASSUME(_CCCL_THREAD_IDX_X < _CCCL_BLOCK_DIM_X);
|
||||
}
|
||||
if constexpr (_BlockExts::static_extent(1) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_BLOCK_DIM_Y == _BlockExts::static_extent(1));
|
||||
_CCCL_ASSUME(_CCCL_THREAD_IDX_Y < _CCCL_BLOCK_DIM_Y);
|
||||
}
|
||||
if constexpr (_BlockExts::static_extent(2) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_BLOCK_DIM_Z == _BlockExts::static_extent(2));
|
||||
_CCCL_ASSUME(_CCCL_THREAD_IDX_Z < _CCCL_BLOCK_DIM_Z);
|
||||
}
|
||||
|
||||
using _GridDesc = typename _Hierarchy::template level_desc_type<grid_level>;
|
||||
using _GridExts = typename _GridDesc::extents_type;
|
||||
|
||||
if constexpr (_Hierarchy::has_level(cluster))
|
||||
{
|
||||
using _ClusterDesc = typename _Hierarchy::template level_desc_type<cluster_level>;
|
||||
using _ClusterExts = typename _ClusterDesc::extents_type;
|
||||
|
||||
// nvc++ doesn't implement clusters yet, so we can just use _CCCL_PTX_ARCH() here. Once the support is there, we can
|
||||
// just add `|| _CCCL_CUDA_COMPILER(NVHPC)`
|
||||
# if _CCCL_PTX_ARCH() >= 900
|
||||
if constexpr (_ClusterExts::static_extent(0) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_DIM_X == _ClusterExts::static_extent(0));
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_RELATIVE_BLOCK_IDX_X < _CCCL_CLUSTER_DIM_X);
|
||||
}
|
||||
if constexpr (_ClusterExts::static_extent(1) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_DIM_Y == _ClusterExts::static_extent(1));
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Y < _CCCL_CLUSTER_DIM_Y);
|
||||
}
|
||||
if constexpr (_ClusterExts::static_extent(2) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_DIM_Z == _ClusterExts::static_extent(2));
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Z < _CCCL_CLUSTER_DIM_Z);
|
||||
}
|
||||
if constexpr (_ClusterExts::static_extent(0) != __dext && _ClusterExts::static_extent(1) != __dext
|
||||
&& _ClusterExts::static_extent(2) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_SIZE_IN_BLOCKS
|
||||
== _ClusterExts::static_extent(0) * _ClusterExts::static_extent(1) * _ClusterExts::static_extent(2));
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_RELATIVE_BLOCK_RANK < _CCCL_CLUSTER_SIZE_IN_BLOCKS);
|
||||
}
|
||||
|
||||
if constexpr (_GridExts::static_extent(0) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_X == _GridExts::static_extent(0));
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_IDX_X < _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_X);
|
||||
}
|
||||
if constexpr (_GridExts::static_extent(1) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Y == _GridExts::static_extent(1));
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_IDX_Y < _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Y);
|
||||
}
|
||||
if constexpr (__dext && _GridExts::static_extent(2) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Z == _GridExts::static_extent(2));
|
||||
_CCCL_ASSUME(_CCCL_CLUSTER_IDX_Z < _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Z);
|
||||
}
|
||||
# endif // _CCCL_PTX_ARCH() >= 900
|
||||
|
||||
if constexpr (_ClusterExts::static_extent(0) != __dext && _GridExts::static_extent(0) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_GRID_DIM_X == _ClusterExts::static_extent(0) * _GridExts::static_extent(0));
|
||||
_CCCL_ASSUME(_CCCL_BLOCK_IDX_X < _CCCL_GRID_DIM_X);
|
||||
}
|
||||
if constexpr (_ClusterExts::static_extent(1) != __dext && _GridExts::static_extent(1) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_GRID_DIM_Y == _ClusterExts::static_extent(1) * _GridExts::static_extent(1));
|
||||
_CCCL_ASSUME(_CCCL_BLOCK_IDX_Y < _CCCL_GRID_DIM_Y);
|
||||
}
|
||||
if constexpr (_ClusterExts::static_extent(2) != __dext && _GridExts::static_extent(2) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_GRID_DIM_Z == _ClusterExts::static_extent(2) * _GridExts::static_extent(2));
|
||||
_CCCL_ASSUME(_CCCL_BLOCK_IDX_Z < _CCCL_GRID_DIM_Z);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if constexpr (_GridExts::static_extent(0) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_GRID_DIM_X == _GridExts::static_extent(0));
|
||||
_CCCL_ASSUME(_CCCL_BLOCK_IDX_X < _CCCL_GRID_DIM_X);
|
||||
}
|
||||
if constexpr (_GridExts::static_extent(1) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_GRID_DIM_Y == _GridExts::static_extent(1));
|
||||
_CCCL_ASSUME(_CCCL_BLOCK_IDX_Y < _CCCL_GRID_DIM_Y);
|
||||
}
|
||||
if constexpr (_GridExts::static_extent(2) != __dext)
|
||||
{
|
||||
_CCCL_ASSUME(_CCCL_GRID_DIM_Z == _GridExts::static_extent(2));
|
||||
_CCCL_ASSUME(_CCCL_BLOCK_IDX_Z < _CCCL_GRID_DIM_Z);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_CCCL_DIAG_POP
|
||||
|
||||
# undef _CCCL_THREAD_IDX_X
|
||||
# undef _CCCL_THREAD_IDX_Y
|
||||
# undef _CCCL_THREAD_IDX_Z
|
||||
# undef _CCCL_BLOCK_DIM_X
|
||||
# undef _CCCL_BLOCK_DIM_Y
|
||||
# undef _CCCL_BLOCK_DIM_Z
|
||||
# undef _CCCL_BLOCK_IDX_X
|
||||
# undef _CCCL_BLOCK_IDX_Y
|
||||
# undef _CCCL_BLOCK_IDX_Z
|
||||
# undef _CCCL_CLUSTER_DIM_X
|
||||
# undef _CCCL_CLUSTER_DIM_Y
|
||||
# undef _CCCL_CLUSTER_DIM_Z
|
||||
# undef _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_X
|
||||
# undef _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Y
|
||||
# undef _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Z
|
||||
# undef _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_X
|
||||
# undef _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Y
|
||||
# undef _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Z
|
||||
# undef _CCCL_CLUSTER_IDX_X
|
||||
# undef _CCCL_CLUSTER_IDX_Y
|
||||
# undef _CCCL_CLUSTER_IDX_Z
|
||||
# undef _CCCL_CLUSTER_RELATIVE_BLOCK_RANK
|
||||
# undef _CCCL_CLUSTER_SIZE_IN_BLOCKS
|
||||
# undef _CCCL_GRID_DIM_X
|
||||
# undef _CCCL_GRID_DIM_Y
|
||||
# undef _CCCL_GRID_DIM_Z
|
||||
|
||||
template <class _Hierarchy, class _Level>
|
||||
[[nodiscard]] _CCCL_DEVICE_API _CCCL_CONSTEVAL unsigned __block_size(::cuda::std::size_t __i) noexcept
|
||||
{
|
||||
static_assert(__is_hierarchy_v<_Hierarchy>);
|
||||
static_assert(__is_hierarchy_level_v<_Level>);
|
||||
using _Desc = typename _Hierarchy::template level_desc_type<_Level>;
|
||||
using _Exts = typename _Desc::extents_type;
|
||||
|
||||
static_assert(_Exts::rank_dynamic() == 0, "this function can be used only with all static extents");
|
||||
return static_cast<unsigned>(_Exts::static_extent(__i));
|
||||
}
|
||||
|
||||
template <class _Hierarchy>
|
||||
[[nodiscard]] _CCCL_DEVICE_API _CCCL_CONSTEVAL unsigned __max_nthreads_per_block() noexcept
|
||||
{
|
||||
static_assert(__is_hierarchy_v<_Hierarchy>);
|
||||
using _BlockDesc = typename _Hierarchy::template level_desc_type<block_level>;
|
||||
using _BlockExts = typename _BlockDesc::extents_type;
|
||||
|
||||
static_assert(_BlockExts::rank_dynamic() == 0, "this function can be used only with all static extents");
|
||||
return static_cast<unsigned>(
|
||||
_BlockExts::static_extent(0) * _BlockExts::static_extent(1) * _BlockExts::static_extent(2));
|
||||
}
|
||||
|
||||
template <class _Kernel, class _Config, class... _Args>
|
||||
inline constexpr bool __invoke_kernel_functor_with_config_v =
|
||||
::cuda::std::is_invocable_v<_Kernel, _Config, ::cuda::std::decay_t<transformed_device_argument_t<_Args>>...>
|
||||
# if _CCCL_CUDA_COMPILER(NVCC)
|
||||
&& !__nv_is_extended_device_lambda_closure_type(_Kernel)
|
||||
# endif
|
||||
;
|
||||
|
||||
// We create 3 kernel functor launchers:
|
||||
// 1. With __block_size__ for cluster launches with compile-time known dims.
|
||||
// 2. With __launch_bounds__ for non-cluster launches with compile-time known block size.
|
||||
// 3. Fallback without any attributes.
|
||||
|
||||
template <class _Config, class _Kernel, class... _Args>
|
||||
__global__ static void
|
||||
// todo(dabayer): Re-enable this once cuda::launch with kernels that were compiled with .blocksareclusters directive is
|
||||
// fixed.
|
||||
//
|
||||
// _CCCL_BLOCK_SIZE((::cuda::__block_size<typename _Config::hierarchy_type, block_level>(0),
|
||||
// ::cuda::__block_size<typename _Config::hierarchy_type, block_level>(1),
|
||||
// ::cuda::__block_size<typename _Config::hierarchy_type, block_level>(2)),
|
||||
// (::cuda::__block_size<typename _Config::hierarchy_type, cluster_level>(0),
|
||||
// ::cuda::__block_size<typename _Config::hierarchy_type, cluster_level>(1),
|
||||
// ::cuda::__block_size<typename _Config::hierarchy_type, cluster_level>(2)))
|
||||
__kernel_launcher_with_block_size(const _CCCL_GRID_CONSTANT _Config __conf, _Kernel __kernel_fn, _Args... __args)
|
||||
{
|
||||
::cuda::__assume_known_info<typename _Config::hierarchy_type>();
|
||||
|
||||
if constexpr (__invoke_kernel_functor_with_config_v<_Kernel, _Config, _Args...>)
|
||||
{
|
||||
__kernel_fn(__conf, __args...);
|
||||
}
|
||||
else
|
||||
{
|
||||
__kernel_fn(__args...);
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Config, class _Kernel, class... _Args>
|
||||
__global__ static void _CCCL_LAUNCH_BOUNDS(::cuda::__max_nthreads_per_block<typename _Config::hierarchy_type>())
|
||||
__kernel_launcher_with_launch_bounds(const _CCCL_GRID_CONSTANT _Config __conf, _Kernel __kernel_fn, _Args... __args)
|
||||
{
|
||||
::cuda::__assume_known_info<typename _Config::hierarchy_type>();
|
||||
|
||||
if constexpr (__invoke_kernel_functor_with_config_v<_Kernel, _Config, _Args...>)
|
||||
{
|
||||
__kernel_fn(__conf, __args...);
|
||||
}
|
||||
else
|
||||
{
|
||||
__kernel_fn(__args...);
|
||||
}
|
||||
}
|
||||
|
||||
template <class _Config, class _Kernel, class... _Args>
|
||||
__global__ static void __kernel_launcher(const _CCCL_GRID_CONSTANT _Config __conf, _Kernel __kernel_fn, _Args... __args)
|
||||
{
|
||||
::cuda::__assume_known_info<typename _Config::hierarchy_type>();
|
||||
|
||||
if constexpr (__invoke_kernel_functor_with_config_v<_Kernel, _Config, _Args...>)
|
||||
{
|
||||
__kernel_fn(__conf, __args...);
|
||||
}
|
||||
else
|
||||
{
|
||||
__kernel_fn(__args...);
|
||||
}
|
||||
}
|
||||
|
||||
// Return void pointer to work around NVCC bug with __restrict__
|
||||
template <class _Kernel, class _Config, class... _Args>
|
||||
[[nodiscard]] _CCCL_API constexpr const void* __get_kernel_launcher() noexcept
|
||||
{
|
||||
using _Hierarchy = typename _Config::hierarchy_type;
|
||||
using _BlockDesc = typename _Hierarchy::template level_desc_type<block_level>;
|
||||
using _BlockExts = typename _BlockDesc::extents_type;
|
||||
|
||||
if constexpr (_BlockExts::rank_dynamic() == 0)
|
||||
{
|
||||
// todo(dabayer): Re-enable the cluster-specific block-size launcher once cuda::launch with kernels compiled with
|
||||
// .blocksareclusters directive is fixed.
|
||||
return reinterpret_cast<const void*>(::cuda::__kernel_launcher_with_launch_bounds<_Config, _Kernel, _Args...>);
|
||||
}
|
||||
else
|
||||
{
|
||||
return reinterpret_cast<const void*>(::cuda::__kernel_launcher<_Config, _Kernel, _Args...>);
|
||||
}
|
||||
}
|
||||
|
||||
# endif // _CCCL_CUDA_COMPILATION()
|
||||
|
||||
[[nodiscard]] _CCCL_HOST_API inline ::CUfunction __get_cufunction_of(const void* __kernel)
|
||||
{
|
||||
::cudaFunction_t __kernel_cufunction{};
|
||||
_CCCL_TRY_CUDA_API(::cudaGetFuncBySymbol, "Failed to get function from symbol", &__kernel_cufunction, __kernel);
|
||||
return (::CUfunction) __kernel_cufunction;
|
||||
}
|
||||
|
||||
_CCCL_HOST_API void inline __do_launch(
|
||||
::cuda::stream_ref __stream, ::CUlaunchConfig& __config, ::CUfunction __kernel, void** __args_ptrs)
|
||||
{
|
||||
__config.hStream = __stream.get();
|
||||
# if defined(_CCCLRT_LAUNCH_CONFIG_TEST)
|
||||
test_launch_kernel_replacement(__config, __kernel, __args_ptrs);
|
||||
# else // ^^^ _CUDAX_LAUNCH_CONFIG_TEST ^^^ / vvv !_CUDAX_LAUNCH_CONFIG_TEST vvv
|
||||
::cuda::__driver::__launchKernel(__config, __kernel, __args_ptrs);
|
||||
# endif // ^^^ !_CUDAX_LAUNCH_CONFIG_TEST ^^^
|
||||
}
|
||||
|
||||
template <typename... _ExpTypes, typename _Dst, typename _Config>
|
||||
_CCCL_HOST_API auto __launch_impl(_Dst&& __dst, _Config __conf, ::CUfunction __kernel, _ExpTypes... __args)
|
||||
{
|
||||
static_assert(!::cuda::std::is_same_v<decltype(__conf.hierarchy()), no_init_t>,
|
||||
"Can't launch a configuration without hierarchy dimensions");
|
||||
|
||||
using _Hierarchy = typename _Config::hierarchy_type;
|
||||
|
||||
::CUlaunchConfig __config{};
|
||||
constexpr bool __has_cluster_level = _Hierarchy::has_level(cluster);
|
||||
constexpr unsigned int __num_attrs_needed = __detail::kernel_config_count_attr_space(__conf) + __has_cluster_level;
|
||||
::CUlaunchAttribute __attrs[__num_attrs_needed == 0 ? 1 : __num_attrs_needed];
|
||||
__config.attrs = &__attrs[0];
|
||||
__config.numAttrs = 0;
|
||||
|
||||
::cudaError_t __status = __detail::apply_kernel_config(__conf, __config, __kernel);
|
||||
if (__status != ::cudaSuccess)
|
||||
{
|
||||
_CCCL_THROW(::cuda::cuda_error, __status, "Failed to prepare a launch configuration");
|
||||
}
|
||||
|
||||
__config.gridDimX = block.dims(grid, __conf).x;
|
||||
__config.gridDimY = block.dims(grid, __conf).y;
|
||||
__config.gridDimZ = block.dims(grid, __conf).z;
|
||||
__config.blockDimX = gpu_thread.dims(block, __conf).x;
|
||||
__config.blockDimY = gpu_thread.dims(block, __conf).y;
|
||||
__config.blockDimZ = gpu_thread.dims(block, __conf).z;
|
||||
|
||||
if constexpr (__has_cluster_level)
|
||||
{
|
||||
::CUlaunchAttribute __cluster_dims_attr{};
|
||||
__cluster_dims_attr.id = ::CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION;
|
||||
__cluster_dims_attr.value.clusterDim.x = block.dims(cluster, __conf).x;
|
||||
__cluster_dims_attr.value.clusterDim.y = block.dims(cluster, __conf).y;
|
||||
__cluster_dims_attr.value.clusterDim.z = block.dims(cluster, __conf).z;
|
||||
__config.attrs[__config.numAttrs++] = __cluster_dims_attr;
|
||||
}
|
||||
|
||||
const void* __pArgs[(sizeof...(__args) > 0) ? sizeof...(__args) : 1]{::cuda::std::addressof(__args)...};
|
||||
return ::cuda::__do_launch(::cuda::std::forward<_Dst>(__dst), __config, __kernel, const_cast<void**>(__pArgs));
|
||||
}
|
||||
|
||||
_CCCL_HOST_API ::cuda::stream_ref inline __stream_or_invalid(::cuda::stream_ref __stream)
|
||||
{
|
||||
return __stream;
|
||||
}
|
||||
|
||||
// cast to stream_ref to avoid instantiating launch_impl for every type
|
||||
// convertible to stream_ref
|
||||
template <typename _Dummy>
|
||||
_CCCL_HOST_API ::cuda::stream_ref __forward_or_cast_to_stream_ref(::cuda::stream_ref __stream)
|
||||
{
|
||||
return __stream;
|
||||
}
|
||||
|
||||
template <typename _Submitter>
|
||||
_CCCL_CONCEPT work_submitter = ::cuda::std::is_convertible_v<_Submitter, ::cuda::stream_ref>;
|
||||
|
||||
# if _CCCL_CUDA_COMPILATION()
|
||||
|
||||
//! @brief Launch a kernel functor with specified configuration and arguments
|
||||
//!
|
||||
//! Launches a kernel functor object on the specified stream and with specified
|
||||
//! configuration. Kernel functor object is a type with __device__ operator().
|
||||
//! Functor might or might not accept the configuration as its first argument.
|
||||
//!
|
||||
//! @par Snippet
|
||||
//! @code
|
||||
//! #include <cstdio>
|
||||
//! #include <cuda/launch>
|
||||
//!
|
||||
//! struct kernel {
|
||||
//! template <typename Configuration>
|
||||
//! __device__ void operator()(Configuration conf, unsigned int
|
||||
//! thread_to_print) {
|
||||
//! if (conf.dims.rank(cuda::thread, cuda::grid) == thread_to_print) {
|
||||
//! printf("Hello from the GPU\n");
|
||||
//! }
|
||||
//! }
|
||||
//! };
|
||||
//!
|
||||
//! void launch_kernel(cuda::stream_ref stream) {
|
||||
//! auto dims = cuda::make_hierarchy(cuda::block_dims<128>(),
|
||||
//! cuda::grid_dims(4)); auto config = cuda::make_config(dims,
|
||||
//! cuda::launch_cooperative());
|
||||
//!
|
||||
//! cuda::launch(stream, config, kernel(), 42);
|
||||
//! }
|
||||
//! @endcode
|
||||
//!
|
||||
//! @param __submitter
|
||||
//! cuda::stream_ref to launch the kernel into
|
||||
//!
|
||||
//! @param __conf
|
||||
//! configuration for this launch
|
||||
//!
|
||||
//! @param __kernel
|
||||
//! kernel functor to be launched
|
||||
//!
|
||||
//! @param __args
|
||||
//! arguments to be passed into the kernel functor
|
||||
_CCCL_TEMPLATE(typename... _Args, typename... _Config, typename _Submitter, typename _Dimensions, typename _Kernel)
|
||||
_CCCL_REQUIRES(work_submitter<_Submitter> _CCCL_AND(!::cuda::std::is_pointer_v<_Kernel>)
|
||||
_CCCL_AND(!::cuda::std::is_function_v<_Kernel>))
|
||||
_CCCL_HOST_API auto launch(_Submitter&& __submitter,
|
||||
const kernel_config<_Dimensions, _Config...>& __conf,
|
||||
const _Kernel& __kernel,
|
||||
_Args&&... __args)
|
||||
{
|
||||
__ensure_current_context __dev_setter{__submitter};
|
||||
auto __combined = __conf.combine_with_default(__kernel);
|
||||
auto __launcher = ::cuda::__get_kernel_launcher<_Kernel,
|
||||
decltype(__combined),
|
||||
::cuda::std::decay_t<transformed_device_argument_t<_Args>>...>();
|
||||
return ::cuda::__launch_impl(
|
||||
cuda::__forward_or_cast_to_stream_ref<_Submitter>(__submitter),
|
||||
__combined,
|
||||
::cuda::__get_cufunction_of(__launcher),
|
||||
__combined,
|
||||
__kernel,
|
||||
launch_transform(::cuda::__stream_or_invalid(__submitter), ::cuda::std::forward<_Args>(__args))...);
|
||||
}
|
||||
|
||||
# endif // _CCCL_CUDA_COMPILATION()
|
||||
|
||||
//! @brief Launch a kernel function with specified configuration and arguments
|
||||
//!
|
||||
//! Launches a kernel function on the specified stream and with specified
|
||||
//! configuration. Kernel function is a function with __global__ annotation.
|
||||
//! Function might or might not accept the configuration as its first argument.
|
||||
//!
|
||||
//! @par Snippet
|
||||
//! @code
|
||||
//! #include <cstdio>
|
||||
//! #include <cuda/launch>
|
||||
//!
|
||||
//! template <typename Configuration>
|
||||
//! __global__ void kernel(Configuration conf, unsigned int thread_to_print) {
|
||||
//! if (conf.dims.rank(cuda::thread, cuda::grid) == thread_to_print) {
|
||||
//! printf("Hello from the GPU\n");
|
||||
//! }
|
||||
//! }
|
||||
//!
|
||||
//! void launch_kernel(cuda::stream_ref stream) {
|
||||
//! auto dims = cuda::make_hierarchy(cuda::block_dims<128>(),
|
||||
//! cuda::grid_dims(4)); auto config = cuda::make_config(dims,
|
||||
//! cuda::launch_cooperative());
|
||||
//!
|
||||
//! cuda::launch(stream, config, kernel<decltype(config)>, 42);
|
||||
//! }
|
||||
//! @endcode
|
||||
//!
|
||||
//! @param __submitter
|
||||
//! cuda::stream_ref to launch the kernel into
|
||||
//!
|
||||
//! @param __conf
|
||||
//! configuration for this launch
|
||||
//!
|
||||
//! @param __kernel
|
||||
//! kernel function to be launched
|
||||
//!
|
||||
//! @param __args
|
||||
//! arguments to be passed into the kernel function
|
||||
//!
|
||||
_CCCL_TEMPLATE(
|
||||
typename... _ExpArgs, typename... _ActArgs, typename _Submitter, typename... _Config, typename _Dimensions)
|
||||
_CCCL_REQUIRES(work_submitter<_Submitter> _CCCL_AND(sizeof...(_ExpArgs) == sizeof...(_ActArgs)))
|
||||
_CCCL_HOST_API auto launch(_Submitter&& __submitter,
|
||||
const kernel_config<_Dimensions, _Config...>& __conf,
|
||||
void (*__kernel)(kernel_config<_Dimensions, _Config...>, _ExpArgs...),
|
||||
_ActArgs&&... __args)
|
||||
{
|
||||
__ensure_current_context __dev_setter{__submitter};
|
||||
return ::cuda::__launch_impl<kernel_config<_Dimensions, _Config...>,
|
||||
_ExpArgs...>(
|
||||
cuda::__forward_or_cast_to_stream_ref<_Submitter>(__submitter), //
|
||||
__conf,
|
||||
::cuda::__get_cufunction_of(reinterpret_cast<const void*>(__kernel)),
|
||||
__conf,
|
||||
launch_transform(::cuda::__stream_or_invalid(__submitter), ::cuda::std::forward<_ActArgs>(__args))...);
|
||||
}
|
||||
|
||||
//! @brief Launch a kernel function with specified configuration and arguments
|
||||
//!
|
||||
//! Launches a kernel function on the specified stream and with specified
|
||||
//! configuration. Kernel function is a function with __global__ annotation.
|
||||
//! Function might or might not accept the configuration as its first argument.
|
||||
//!
|
||||
//! @par Snippet
|
||||
//! @code
|
||||
//! #include <cstdio>
|
||||
//! #include <cuda/launch>
|
||||
//!
|
||||
//! template <typename Configuration>
|
||||
//! __global__ void kernel(Configuration conf, unsigned int thread_to_print) {
|
||||
//! if (conf.dims.rank(cuda::thread, cuda::grid) == thread_to_print) {
|
||||
//! printf("Hello from the GPU\n");
|
||||
//! }
|
||||
//! }
|
||||
//!
|
||||
//! void launch_kernel(cuda::stream_ref stream) {
|
||||
//! auto dims = cuda::make_hierarchy(cuda::block_dims<128>(),
|
||||
//! cuda::grid_dims(4)); auto config = cuda::make_config(dims,
|
||||
//! cuda::launch_cooperative());
|
||||
//!
|
||||
//! cuda::launch(stream, config, kernel<decltype(config)>, 42);
|
||||
//! }
|
||||
//! @endcode
|
||||
//!
|
||||
//! @param __submitter
|
||||
//! cuda::stream_ref to launch the kernel into
|
||||
//!
|
||||
//! @param __conf
|
||||
//! configuration for this launch
|
||||
//!
|
||||
//! @param __kernel
|
||||
//! kernel function to be launched
|
||||
//!
|
||||
//! @param __args
|
||||
//! arguments to be passed into the kernel function
|
||||
_CCCL_TEMPLATE(
|
||||
typename... _ExpArgs, typename... _ActArgs, typename _Submitter, typename... _Config, typename _Dimensions)
|
||||
_CCCL_REQUIRES(work_submitter<_Submitter> _CCCL_AND(sizeof...(_ExpArgs) == sizeof...(_ActArgs)))
|
||||
_CCCL_HOST_API auto launch(_Submitter&& __submitter,
|
||||
const kernel_config<_Dimensions, _Config...>& __conf,
|
||||
void (*__kernel)(_ExpArgs...),
|
||||
_ActArgs&&... __args)
|
||||
{
|
||||
__ensure_current_context __dev_setter{__submitter};
|
||||
return ::cuda::__launch_impl<_ExpArgs...>(
|
||||
cuda::__forward_or_cast_to_stream_ref<_Submitter>(__submitter), //
|
||||
__conf,
|
||||
::cuda::__get_cufunction_of(reinterpret_cast<const void*>(__kernel)),
|
||||
launch_transform(::cuda::__stream_or_invalid(__submitter), ::cuda::std::forward<_ActArgs>(__args))...);
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
||||
|
||||
#endif // _CUDA___LAUNCH_LAUNCH_H
|
||||
Reference in New Issue
Block a user