56fd68e7dd
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
629 lines
26 KiB
C++
629 lines
26 KiB
C++
//===----------------------------------------------------------------------===//
|
|
//
|
|
// Part of libcu++, the C++ Standard Library for your entire system,
|
|
// under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#ifndef _CUDA___LAUNCH_LAUNCH_H
|
|
#define _CUDA___LAUNCH_LAUNCH_H
|
|
|
|
#include <cuda/std/detail/__config>
|
|
|
|
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
|
# pragma GCC system_header
|
|
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
|
# pragma clang system_header
|
|
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
|
# pragma system_header
|
|
#endif // no system header
|
|
|
|
#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
|
|
|
# include <cuda/__driver/driver_api.h>
|
|
# include <cuda/__hierarchy/hierarchy_levels.h>
|
|
# include <cuda/__hierarchy/traits.h>
|
|
# include <cuda/__launch/configuration.h>
|
|
# include <cuda/__runtime/api_wrapper.h>
|
|
# include <cuda/__runtime/ensure_current_context.h>
|
|
# include <cuda/__stream/launch_transform.h>
|
|
# include <cuda/__stream/stream_ref.h>
|
|
# include <cuda/std/__exception/cuda_error.h>
|
|
# include <cuda/std/__exception/exception_macros.h>
|
|
# include <cuda/std/__type_traits/is_function.h>
|
|
# include <cuda/std/__type_traits/is_pointer.h>
|
|
# include <cuda/std/__type_traits/type_identity.h>
|
|
# include <cuda/std/__utility/forward.h>
|
|
# include <cuda/std/__utility/pod_tuple.h>
|
|
|
|
# include <cuda/std/__cccl/prologue.h>
|
|
|
|
_CCCL_BEGIN_NAMESPACE_CUDA
|
|
|
|
# if _CCCL_CUDA_COMPILATION()
|
|
|
|
// clang-cuda replaces the variables with direct nvvm sreg calls, so we want to assume their return values directly.
|
|
# if _CCCL_CUDA_COMPILER(CLANG)
|
|
# define _CCCL_THREAD_IDX_X ::__nvvm_read_ptx_sreg_tid_x()
|
|
# define _CCCL_THREAD_IDX_Y ::__nvvm_read_ptx_sreg_tid_y()
|
|
# define _CCCL_THREAD_IDX_Z ::__nvvm_read_ptx_sreg_tid_z()
|
|
# define _CCCL_BLOCK_DIM_X ::__nvvm_read_ptx_sreg_ntid_x()
|
|
# define _CCCL_BLOCK_DIM_Y ::__nvvm_read_ptx_sreg_ntid_y()
|
|
# define _CCCL_BLOCK_DIM_Z ::__nvvm_read_ptx_sreg_ntid_z()
|
|
# define _CCCL_BLOCK_IDX_X ::__nvvm_read_ptx_sreg_ctaid_x()
|
|
# define _CCCL_BLOCK_IDX_Y ::__nvvm_read_ptx_sreg_ctaid_y()
|
|
# define _CCCL_BLOCK_IDX_Z ::__nvvm_read_ptx_sreg_ctaid_z()
|
|
# define _CCCL_CLUSTER_DIM_X ::__nvvm_read_ptx_sreg_cluster_nctaid_x()
|
|
# define _CCCL_CLUSTER_DIM_Y ::__nvvm_read_ptx_sreg_cluster_nctaid_y()
|
|
# define _CCCL_CLUSTER_DIM_Z ::__nvvm_read_ptx_sreg_cluster_nctaid_z()
|
|
# define _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_X ::__nvvm_read_ptx_sreg_cluster_ctaid_x()
|
|
# define _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Y ::__nvvm_read_ptx_sreg_cluster_ctaid_y()
|
|
# define _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Z ::__nvvm_read_ptx_sreg_cluster_ctaid_z()
|
|
# define _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_X ::__nvvm_read_ptx_sreg_nclusterid_x()
|
|
# define _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Y ::__nvvm_read_ptx_sreg_nclusterid_y()
|
|
# define _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Z ::__nvvm_read_ptx_sreg_nclusterid_z()
|
|
# define _CCCL_CLUSTER_IDX_X ::__nvvm_read_ptx_sreg_clusterid_x()
|
|
# define _CCCL_CLUSTER_IDX_Y ::__nvvm_read_ptx_sreg_clusterid_y()
|
|
# define _CCCL_CLUSTER_IDX_Z ::__nvvm_read_ptx_sreg_clusterid_z()
|
|
# define _CCCL_CLUSTER_RELATIVE_BLOCK_RANK ::__nvvm_read_ptx_sreg_cluster_ctarank()
|
|
# define _CCCL_CLUSTER_SIZE_IN_BLOCKS ::__nvvm_read_ptx_sreg_cluster_nctarank()
|
|
# define _CCCL_GRID_DIM_X ::__nvvm_read_ptx_sreg_nctaid_x()
|
|
# define _CCCL_GRID_DIM_Y ::__nvvm_read_ptx_sreg_nctaid_y()
|
|
# define _CCCL_GRID_DIM_Z ::__nvvm_read_ptx_sreg_nctaid_z()
|
|
# else // ^^^ _CCCL_CUDA_COMPILER(CLANG) ^^^ / vvv !_CCCL_CUDA_COMPILER(CLANG) vvv
|
|
# define _CCCL_THREAD_IDX_X threadIdx.x
|
|
# define _CCCL_THREAD_IDX_Y threadIdx.y
|
|
# define _CCCL_THREAD_IDX_Z threadIdx.z
|
|
# define _CCCL_BLOCK_DIM_X blockDim.x
|
|
# define _CCCL_BLOCK_DIM_Y blockDim.y
|
|
# define _CCCL_BLOCK_DIM_Z blockDim.z
|
|
# define _CCCL_BLOCK_IDX_X blockIdx.x
|
|
# define _CCCL_BLOCK_IDX_Y blockIdx.y
|
|
# define _CCCL_BLOCK_IDX_Z blockIdx.z
|
|
# define _CCCL_CLUSTER_DIM_X ::__clusterDim().x
|
|
# define _CCCL_CLUSTER_DIM_Y ::__clusterDim().y
|
|
# define _CCCL_CLUSTER_DIM_Z ::__clusterDim().z
|
|
# define _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_X ::__clusterRelativeBlockIdx().x
|
|
# define _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Y ::__clusterRelativeBlockIdx().y
|
|
# define _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Z ::__clusterRelativeBlockIdx().z
|
|
# define _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_X ::__clusterGridDimInClusters().x
|
|
# define _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Y ::__clusterGridDimInClusters().y
|
|
# define _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Z ::__clusterGridDimInClusters().z
|
|
# define _CCCL_CLUSTER_IDX_X ::__clusterIdx().x
|
|
# define _CCCL_CLUSTER_IDX_Y ::__clusterIdx().y
|
|
# define _CCCL_CLUSTER_IDX_Z ::__clusterIdx().z
|
|
# define _CCCL_CLUSTER_RELATIVE_BLOCK_RANK ::__clusterRelativeBlockRank()
|
|
# define _CCCL_CLUSTER_SIZE_IN_BLOCKS ::__clusterSizeInBlocks()
|
|
# define _CCCL_GRID_DIM_X gridDim.x
|
|
# define _CCCL_GRID_DIM_Y gridDim.y
|
|
# define _CCCL_GRID_DIM_Z gridDim.z
|
|
# endif // ^^^ !_CCCL_CUDA_COMPILER(CLANG) ^^^
|
|
|
|
// clang-cuda sometimes warns about the assumption being ignored because it contains (potential) side-effects. We can
|
|
// just suppress it, because in the worst case, the assumption will be just ignored.
|
|
// note(dabayer): I haven't found out when exactly the assumption fails.
|
|
_CCCL_DIAG_PUSH
|
|
_CCCL_DIAG_SUPPRESS_CLANG("-Wassume")
|
|
|
|
template <class _Hierarchy>
|
|
_CCCL_DEVICE_API _CCCL_FORCEINLINE void __assume_known_info() noexcept
|
|
{
|
|
static_assert(__is_hierarchy_v<_Hierarchy>);
|
|
|
|
constexpr auto __dext = ::cuda::std::dynamic_extent;
|
|
|
|
using _BlockDesc = typename _Hierarchy::template level_desc_type<block_level>;
|
|
using _BlockExts = typename _BlockDesc::extents_type;
|
|
|
|
if constexpr (_BlockExts::static_extent(0) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_BLOCK_DIM_X == _BlockExts::static_extent(0));
|
|
_CCCL_ASSUME(_CCCL_THREAD_IDX_X < _CCCL_BLOCK_DIM_X);
|
|
}
|
|
if constexpr (_BlockExts::static_extent(1) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_BLOCK_DIM_Y == _BlockExts::static_extent(1));
|
|
_CCCL_ASSUME(_CCCL_THREAD_IDX_Y < _CCCL_BLOCK_DIM_Y);
|
|
}
|
|
if constexpr (_BlockExts::static_extent(2) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_BLOCK_DIM_Z == _BlockExts::static_extent(2));
|
|
_CCCL_ASSUME(_CCCL_THREAD_IDX_Z < _CCCL_BLOCK_DIM_Z);
|
|
}
|
|
|
|
using _GridDesc = typename _Hierarchy::template level_desc_type<grid_level>;
|
|
using _GridExts = typename _GridDesc::extents_type;
|
|
|
|
if constexpr (_Hierarchy::has_level(cluster))
|
|
{
|
|
using _ClusterDesc = typename _Hierarchy::template level_desc_type<cluster_level>;
|
|
using _ClusterExts = typename _ClusterDesc::extents_type;
|
|
|
|
// nvc++ doesn't implement clusters yet, so we can just use _CCCL_PTX_ARCH() here. Once the support is there, we can
|
|
// just add `|| _CCCL_CUDA_COMPILER(NVHPC)`
|
|
# if _CCCL_PTX_ARCH() >= 900
|
|
if constexpr (_ClusterExts::static_extent(0) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_DIM_X == _ClusterExts::static_extent(0));
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_RELATIVE_BLOCK_IDX_X < _CCCL_CLUSTER_DIM_X);
|
|
}
|
|
if constexpr (_ClusterExts::static_extent(1) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_DIM_Y == _ClusterExts::static_extent(1));
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Y < _CCCL_CLUSTER_DIM_Y);
|
|
}
|
|
if constexpr (_ClusterExts::static_extent(2) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_DIM_Z == _ClusterExts::static_extent(2));
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Z < _CCCL_CLUSTER_DIM_Z);
|
|
}
|
|
if constexpr (_ClusterExts::static_extent(0) != __dext && _ClusterExts::static_extent(1) != __dext
|
|
&& _ClusterExts::static_extent(2) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_SIZE_IN_BLOCKS
|
|
== _ClusterExts::static_extent(0) * _ClusterExts::static_extent(1) * _ClusterExts::static_extent(2));
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_RELATIVE_BLOCK_RANK < _CCCL_CLUSTER_SIZE_IN_BLOCKS);
|
|
}
|
|
|
|
if constexpr (_GridExts::static_extent(0) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_X == _GridExts::static_extent(0));
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_IDX_X < _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_X);
|
|
}
|
|
if constexpr (_GridExts::static_extent(1) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Y == _GridExts::static_extent(1));
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_IDX_Y < _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Y);
|
|
}
|
|
if constexpr (__dext && _GridExts::static_extent(2) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Z == _GridExts::static_extent(2));
|
|
_CCCL_ASSUME(_CCCL_CLUSTER_IDX_Z < _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Z);
|
|
}
|
|
# endif // _CCCL_PTX_ARCH() >= 900
|
|
|
|
if constexpr (_ClusterExts::static_extent(0) != __dext && _GridExts::static_extent(0) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_GRID_DIM_X == _ClusterExts::static_extent(0) * _GridExts::static_extent(0));
|
|
_CCCL_ASSUME(_CCCL_BLOCK_IDX_X < _CCCL_GRID_DIM_X);
|
|
}
|
|
if constexpr (_ClusterExts::static_extent(1) != __dext && _GridExts::static_extent(1) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_GRID_DIM_Y == _ClusterExts::static_extent(1) * _GridExts::static_extent(1));
|
|
_CCCL_ASSUME(_CCCL_BLOCK_IDX_Y < _CCCL_GRID_DIM_Y);
|
|
}
|
|
if constexpr (_ClusterExts::static_extent(2) != __dext && _GridExts::static_extent(2) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_GRID_DIM_Z == _ClusterExts::static_extent(2) * _GridExts::static_extent(2));
|
|
_CCCL_ASSUME(_CCCL_BLOCK_IDX_Z < _CCCL_GRID_DIM_Z);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
if constexpr (_GridExts::static_extent(0) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_GRID_DIM_X == _GridExts::static_extent(0));
|
|
_CCCL_ASSUME(_CCCL_BLOCK_IDX_X < _CCCL_GRID_DIM_X);
|
|
}
|
|
if constexpr (_GridExts::static_extent(1) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_GRID_DIM_Y == _GridExts::static_extent(1));
|
|
_CCCL_ASSUME(_CCCL_BLOCK_IDX_Y < _CCCL_GRID_DIM_Y);
|
|
}
|
|
if constexpr (_GridExts::static_extent(2) != __dext)
|
|
{
|
|
_CCCL_ASSUME(_CCCL_GRID_DIM_Z == _GridExts::static_extent(2));
|
|
_CCCL_ASSUME(_CCCL_BLOCK_IDX_Z < _CCCL_GRID_DIM_Z);
|
|
}
|
|
}
|
|
}
|
|
|
|
_CCCL_DIAG_POP
|
|
|
|
# undef _CCCL_THREAD_IDX_X
|
|
# undef _CCCL_THREAD_IDX_Y
|
|
# undef _CCCL_THREAD_IDX_Z
|
|
# undef _CCCL_BLOCK_DIM_X
|
|
# undef _CCCL_BLOCK_DIM_Y
|
|
# undef _CCCL_BLOCK_DIM_Z
|
|
# undef _CCCL_BLOCK_IDX_X
|
|
# undef _CCCL_BLOCK_IDX_Y
|
|
# undef _CCCL_BLOCK_IDX_Z
|
|
# undef _CCCL_CLUSTER_DIM_X
|
|
# undef _CCCL_CLUSTER_DIM_Y
|
|
# undef _CCCL_CLUSTER_DIM_Z
|
|
# undef _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_X
|
|
# undef _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Y
|
|
# undef _CCCL_CLUSTER_RELATIVE_BLOCK_IDX_Z
|
|
# undef _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_X
|
|
# undef _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Y
|
|
# undef _CCCL_CLUSTER_GRID_DIM_IN_CLUSTERS_Z
|
|
# undef _CCCL_CLUSTER_IDX_X
|
|
# undef _CCCL_CLUSTER_IDX_Y
|
|
# undef _CCCL_CLUSTER_IDX_Z
|
|
# undef _CCCL_CLUSTER_RELATIVE_BLOCK_RANK
|
|
# undef _CCCL_CLUSTER_SIZE_IN_BLOCKS
|
|
# undef _CCCL_GRID_DIM_X
|
|
# undef _CCCL_GRID_DIM_Y
|
|
# undef _CCCL_GRID_DIM_Z
|
|
|
|
template <class _Hierarchy, class _Level>
|
|
[[nodiscard]] _CCCL_DEVICE_API _CCCL_CONSTEVAL unsigned __block_size(::cuda::std::size_t __i) noexcept
|
|
{
|
|
static_assert(__is_hierarchy_v<_Hierarchy>);
|
|
static_assert(__is_hierarchy_level_v<_Level>);
|
|
using _Desc = typename _Hierarchy::template level_desc_type<_Level>;
|
|
using _Exts = typename _Desc::extents_type;
|
|
|
|
static_assert(_Exts::rank_dynamic() == 0, "this function can be used only with all static extents");
|
|
return static_cast<unsigned>(_Exts::static_extent(__i));
|
|
}
|
|
|
|
template <class _Hierarchy>
|
|
[[nodiscard]] _CCCL_DEVICE_API _CCCL_CONSTEVAL unsigned __max_nthreads_per_block() noexcept
|
|
{
|
|
static_assert(__is_hierarchy_v<_Hierarchy>);
|
|
using _BlockDesc = typename _Hierarchy::template level_desc_type<block_level>;
|
|
using _BlockExts = typename _BlockDesc::extents_type;
|
|
|
|
static_assert(_BlockExts::rank_dynamic() == 0, "this function can be used only with all static extents");
|
|
return static_cast<unsigned>(
|
|
_BlockExts::static_extent(0) * _BlockExts::static_extent(1) * _BlockExts::static_extent(2));
|
|
}
|
|
|
|
template <class _Kernel, class _Config, class... _Args>
|
|
inline constexpr bool __invoke_kernel_functor_with_config_v =
|
|
::cuda::std::is_invocable_v<_Kernel, _Config, ::cuda::std::decay_t<transformed_device_argument_t<_Args>>...>
|
|
# if _CCCL_CUDA_COMPILER(NVCC)
|
|
&& !__nv_is_extended_device_lambda_closure_type(_Kernel)
|
|
# endif
|
|
;
|
|
|
|
// We create 3 kernel functor launchers:
|
|
// 1. With __block_size__ for cluster launches with compile-time known dims.
|
|
// 2. With __launch_bounds__ for non-cluster launches with compile-time known block size.
|
|
// 3. Fallback without any attributes.
|
|
|
|
template <class _Config, class _Kernel, class... _Args>
|
|
__global__ static void
|
|
// todo(dabayer): Re-enable this once cuda::launch with kernels that were compiled with .blocksareclusters directive is
|
|
// fixed.
|
|
//
|
|
// _CCCL_BLOCK_SIZE((::cuda::__block_size<typename _Config::hierarchy_type, block_level>(0),
|
|
// ::cuda::__block_size<typename _Config::hierarchy_type, block_level>(1),
|
|
// ::cuda::__block_size<typename _Config::hierarchy_type, block_level>(2)),
|
|
// (::cuda::__block_size<typename _Config::hierarchy_type, cluster_level>(0),
|
|
// ::cuda::__block_size<typename _Config::hierarchy_type, cluster_level>(1),
|
|
// ::cuda::__block_size<typename _Config::hierarchy_type, cluster_level>(2)))
|
|
__kernel_launcher_with_block_size(const _CCCL_GRID_CONSTANT _Config __conf, _Kernel __kernel_fn, _Args... __args)
|
|
{
|
|
::cuda::__assume_known_info<typename _Config::hierarchy_type>();
|
|
|
|
if constexpr (__invoke_kernel_functor_with_config_v<_Kernel, _Config, _Args...>)
|
|
{
|
|
__kernel_fn(__conf, __args...);
|
|
}
|
|
else
|
|
{
|
|
__kernel_fn(__args...);
|
|
}
|
|
}
|
|
|
|
template <class _Config, class _Kernel, class... _Args>
|
|
__global__ static void _CCCL_LAUNCH_BOUNDS(::cuda::__max_nthreads_per_block<typename _Config::hierarchy_type>())
|
|
__kernel_launcher_with_launch_bounds(const _CCCL_GRID_CONSTANT _Config __conf, _Kernel __kernel_fn, _Args... __args)
|
|
{
|
|
::cuda::__assume_known_info<typename _Config::hierarchy_type>();
|
|
|
|
if constexpr (__invoke_kernel_functor_with_config_v<_Kernel, _Config, _Args...>)
|
|
{
|
|
__kernel_fn(__conf, __args...);
|
|
}
|
|
else
|
|
{
|
|
__kernel_fn(__args...);
|
|
}
|
|
}
|
|
|
|
template <class _Config, class _Kernel, class... _Args>
|
|
__global__ static void __kernel_launcher(const _CCCL_GRID_CONSTANT _Config __conf, _Kernel __kernel_fn, _Args... __args)
|
|
{
|
|
::cuda::__assume_known_info<typename _Config::hierarchy_type>();
|
|
|
|
if constexpr (__invoke_kernel_functor_with_config_v<_Kernel, _Config, _Args...>)
|
|
{
|
|
__kernel_fn(__conf, __args...);
|
|
}
|
|
else
|
|
{
|
|
__kernel_fn(__args...);
|
|
}
|
|
}
|
|
|
|
// Return void pointer to work around NVCC bug with __restrict__
|
|
template <class _Kernel, class _Config, class... _Args>
|
|
[[nodiscard]] _CCCL_API constexpr const void* __get_kernel_launcher() noexcept
|
|
{
|
|
using _Hierarchy = typename _Config::hierarchy_type;
|
|
using _BlockDesc = typename _Hierarchy::template level_desc_type<block_level>;
|
|
using _BlockExts = typename _BlockDesc::extents_type;
|
|
|
|
if constexpr (_BlockExts::rank_dynamic() == 0)
|
|
{
|
|
// todo(dabayer): Re-enable the cluster-specific block-size launcher once cuda::launch with kernels compiled with
|
|
// .blocksareclusters directive is fixed.
|
|
return reinterpret_cast<const void*>(::cuda::__kernel_launcher_with_launch_bounds<_Config, _Kernel, _Args...>);
|
|
}
|
|
else
|
|
{
|
|
return reinterpret_cast<const void*>(::cuda::__kernel_launcher<_Config, _Kernel, _Args...>);
|
|
}
|
|
}
|
|
|
|
# endif // _CCCL_CUDA_COMPILATION()
|
|
|
|
[[nodiscard]] _CCCL_HOST_API inline ::CUfunction __get_cufunction_of(const void* __kernel)
|
|
{
|
|
::cudaFunction_t __kernel_cufunction{};
|
|
_CCCL_TRY_CUDA_API(::cudaGetFuncBySymbol, "Failed to get function from symbol", &__kernel_cufunction, __kernel);
|
|
return (::CUfunction) __kernel_cufunction;
|
|
}
|
|
|
|
_CCCL_HOST_API void inline __do_launch(
|
|
::cuda::stream_ref __stream, ::CUlaunchConfig& __config, ::CUfunction __kernel, void** __args_ptrs)
|
|
{
|
|
__config.hStream = __stream.get();
|
|
# if defined(_CCCLRT_LAUNCH_CONFIG_TEST)
|
|
test_launch_kernel_replacement(__config, __kernel, __args_ptrs);
|
|
# else // ^^^ _CUDAX_LAUNCH_CONFIG_TEST ^^^ / vvv !_CUDAX_LAUNCH_CONFIG_TEST vvv
|
|
::cuda::__driver::__launchKernel(__config, __kernel, __args_ptrs);
|
|
# endif // ^^^ !_CUDAX_LAUNCH_CONFIG_TEST ^^^
|
|
}
|
|
|
|
template <typename... _ExpTypes, typename _Dst, typename _Config>
|
|
_CCCL_HOST_API auto __launch_impl(_Dst&& __dst, _Config __conf, ::CUfunction __kernel, _ExpTypes... __args)
|
|
{
|
|
static_assert(!::cuda::std::is_same_v<decltype(__conf.hierarchy()), no_init_t>,
|
|
"Can't launch a configuration without hierarchy dimensions");
|
|
|
|
using _Hierarchy = typename _Config::hierarchy_type;
|
|
|
|
::CUlaunchConfig __config{};
|
|
constexpr bool __has_cluster_level = _Hierarchy::has_level(cluster);
|
|
constexpr unsigned int __num_attrs_needed = __detail::kernel_config_count_attr_space(__conf) + __has_cluster_level;
|
|
::CUlaunchAttribute __attrs[__num_attrs_needed == 0 ? 1 : __num_attrs_needed];
|
|
__config.attrs = &__attrs[0];
|
|
__config.numAttrs = 0;
|
|
|
|
::cudaError_t __status = __detail::apply_kernel_config(__conf, __config, __kernel);
|
|
if (__status != ::cudaSuccess)
|
|
{
|
|
_CCCL_THROW(::cuda::cuda_error, __status, "Failed to prepare a launch configuration");
|
|
}
|
|
|
|
__config.gridDimX = block.dims(grid, __conf).x;
|
|
__config.gridDimY = block.dims(grid, __conf).y;
|
|
__config.gridDimZ = block.dims(grid, __conf).z;
|
|
__config.blockDimX = gpu_thread.dims(block, __conf).x;
|
|
__config.blockDimY = gpu_thread.dims(block, __conf).y;
|
|
__config.blockDimZ = gpu_thread.dims(block, __conf).z;
|
|
|
|
if constexpr (__has_cluster_level)
|
|
{
|
|
::CUlaunchAttribute __cluster_dims_attr{};
|
|
__cluster_dims_attr.id = ::CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION;
|
|
__cluster_dims_attr.value.clusterDim.x = block.dims(cluster, __conf).x;
|
|
__cluster_dims_attr.value.clusterDim.y = block.dims(cluster, __conf).y;
|
|
__cluster_dims_attr.value.clusterDim.z = block.dims(cluster, __conf).z;
|
|
__config.attrs[__config.numAttrs++] = __cluster_dims_attr;
|
|
}
|
|
|
|
const void* __pArgs[(sizeof...(__args) > 0) ? sizeof...(__args) : 1]{::cuda::std::addressof(__args)...};
|
|
return ::cuda::__do_launch(::cuda::std::forward<_Dst>(__dst), __config, __kernel, const_cast<void**>(__pArgs));
|
|
}
|
|
|
|
_CCCL_HOST_API ::cuda::stream_ref inline __stream_or_invalid(::cuda::stream_ref __stream)
|
|
{
|
|
return __stream;
|
|
}
|
|
|
|
// cast to stream_ref to avoid instantiating launch_impl for every type
|
|
// convertible to stream_ref
|
|
template <typename _Dummy>
|
|
_CCCL_HOST_API ::cuda::stream_ref __forward_or_cast_to_stream_ref(::cuda::stream_ref __stream)
|
|
{
|
|
return __stream;
|
|
}
|
|
|
|
template <typename _Submitter>
|
|
_CCCL_CONCEPT work_submitter = ::cuda::std::is_convertible_v<_Submitter, ::cuda::stream_ref>;
|
|
|
|
# if _CCCL_CUDA_COMPILATION()
|
|
|
|
//! @brief Launch a kernel functor with specified configuration and arguments
|
|
//!
|
|
//! Launches a kernel functor object on the specified stream and with specified
|
|
//! configuration. Kernel functor object is a type with __device__ operator().
|
|
//! Functor might or might not accept the configuration as its first argument.
|
|
//!
|
|
//! @par Snippet
|
|
//! @code
|
|
//! #include <cstdio>
|
|
//! #include <cuda/launch>
|
|
//!
|
|
//! struct kernel {
|
|
//! template <typename Configuration>
|
|
//! __device__ void operator()(Configuration conf, unsigned int
|
|
//! thread_to_print) {
|
|
//! if (conf.dims.rank(cuda::thread, cuda::grid) == thread_to_print) {
|
|
//! printf("Hello from the GPU\n");
|
|
//! }
|
|
//! }
|
|
//! };
|
|
//!
|
|
//! void launch_kernel(cuda::stream_ref stream) {
|
|
//! auto dims = cuda::make_hierarchy(cuda::block_dims<128>(),
|
|
//! cuda::grid_dims(4)); auto config = cuda::make_config(dims,
|
|
//! cuda::launch_cooperative());
|
|
//!
|
|
//! cuda::launch(stream, config, kernel(), 42);
|
|
//! }
|
|
//! @endcode
|
|
//!
|
|
//! @param __submitter
|
|
//! cuda::stream_ref to launch the kernel into
|
|
//!
|
|
//! @param __conf
|
|
//! configuration for this launch
|
|
//!
|
|
//! @param __kernel
|
|
//! kernel functor to be launched
|
|
//!
|
|
//! @param __args
|
|
//! arguments to be passed into the kernel functor
|
|
_CCCL_TEMPLATE(typename... _Args, typename... _Config, typename _Submitter, typename _Dimensions, typename _Kernel)
|
|
_CCCL_REQUIRES(work_submitter<_Submitter> _CCCL_AND(!::cuda::std::is_pointer_v<_Kernel>)
|
|
_CCCL_AND(!::cuda::std::is_function_v<_Kernel>))
|
|
_CCCL_HOST_API auto launch(_Submitter&& __submitter,
|
|
const kernel_config<_Dimensions, _Config...>& __conf,
|
|
const _Kernel& __kernel,
|
|
_Args&&... __args)
|
|
{
|
|
__ensure_current_context __dev_setter{__submitter};
|
|
auto __combined = __conf.combine_with_default(__kernel);
|
|
auto __launcher = ::cuda::__get_kernel_launcher<_Kernel,
|
|
decltype(__combined),
|
|
::cuda::std::decay_t<transformed_device_argument_t<_Args>>...>();
|
|
return ::cuda::__launch_impl(
|
|
cuda::__forward_or_cast_to_stream_ref<_Submitter>(__submitter),
|
|
__combined,
|
|
::cuda::__get_cufunction_of(__launcher),
|
|
__combined,
|
|
__kernel,
|
|
launch_transform(::cuda::__stream_or_invalid(__submitter), ::cuda::std::forward<_Args>(__args))...);
|
|
}
|
|
|
|
# endif // _CCCL_CUDA_COMPILATION()
|
|
|
|
//! @brief Launch a kernel function with specified configuration and arguments
|
|
//!
|
|
//! Launches a kernel function on the specified stream and with specified
|
|
//! configuration. Kernel function is a function with __global__ annotation.
|
|
//! Function might or might not accept the configuration as its first argument.
|
|
//!
|
|
//! @par Snippet
|
|
//! @code
|
|
//! #include <cstdio>
|
|
//! #include <cuda/launch>
|
|
//!
|
|
//! template <typename Configuration>
|
|
//! __global__ void kernel(Configuration conf, unsigned int thread_to_print) {
|
|
//! if (conf.dims.rank(cuda::thread, cuda::grid) == thread_to_print) {
|
|
//! printf("Hello from the GPU\n");
|
|
//! }
|
|
//! }
|
|
//!
|
|
//! void launch_kernel(cuda::stream_ref stream) {
|
|
//! auto dims = cuda::make_hierarchy(cuda::block_dims<128>(),
|
|
//! cuda::grid_dims(4)); auto config = cuda::make_config(dims,
|
|
//! cuda::launch_cooperative());
|
|
//!
|
|
//! cuda::launch(stream, config, kernel<decltype(config)>, 42);
|
|
//! }
|
|
//! @endcode
|
|
//!
|
|
//! @param __submitter
|
|
//! cuda::stream_ref to launch the kernel into
|
|
//!
|
|
//! @param __conf
|
|
//! configuration for this launch
|
|
//!
|
|
//! @param __kernel
|
|
//! kernel function to be launched
|
|
//!
|
|
//! @param __args
|
|
//! arguments to be passed into the kernel function
|
|
//!
|
|
_CCCL_TEMPLATE(
|
|
typename... _ExpArgs, typename... _ActArgs, typename _Submitter, typename... _Config, typename _Dimensions)
|
|
_CCCL_REQUIRES(work_submitter<_Submitter> _CCCL_AND(sizeof...(_ExpArgs) == sizeof...(_ActArgs)))
|
|
_CCCL_HOST_API auto launch(_Submitter&& __submitter,
|
|
const kernel_config<_Dimensions, _Config...>& __conf,
|
|
void (*__kernel)(kernel_config<_Dimensions, _Config...>, _ExpArgs...),
|
|
_ActArgs&&... __args)
|
|
{
|
|
__ensure_current_context __dev_setter{__submitter};
|
|
return ::cuda::__launch_impl<kernel_config<_Dimensions, _Config...>,
|
|
_ExpArgs...>(
|
|
cuda::__forward_or_cast_to_stream_ref<_Submitter>(__submitter), //
|
|
__conf,
|
|
::cuda::__get_cufunction_of(reinterpret_cast<const void*>(__kernel)),
|
|
__conf,
|
|
launch_transform(::cuda::__stream_or_invalid(__submitter), ::cuda::std::forward<_ActArgs>(__args))...);
|
|
}
|
|
|
|
//! @brief Launch a kernel function with specified configuration and arguments
|
|
//!
|
|
//! Launches a kernel function on the specified stream and with specified
|
|
//! configuration. Kernel function is a function with __global__ annotation.
|
|
//! Function might or might not accept the configuration as its first argument.
|
|
//!
|
|
//! @par Snippet
|
|
//! @code
|
|
//! #include <cstdio>
|
|
//! #include <cuda/launch>
|
|
//!
|
|
//! template <typename Configuration>
|
|
//! __global__ void kernel(Configuration conf, unsigned int thread_to_print) {
|
|
//! if (conf.dims.rank(cuda::thread, cuda::grid) == thread_to_print) {
|
|
//! printf("Hello from the GPU\n");
|
|
//! }
|
|
//! }
|
|
//!
|
|
//! void launch_kernel(cuda::stream_ref stream) {
|
|
//! auto dims = cuda::make_hierarchy(cuda::block_dims<128>(),
|
|
//! cuda::grid_dims(4)); auto config = cuda::make_config(dims,
|
|
//! cuda::launch_cooperative());
|
|
//!
|
|
//! cuda::launch(stream, config, kernel<decltype(config)>, 42);
|
|
//! }
|
|
//! @endcode
|
|
//!
|
|
//! @param __submitter
|
|
//! cuda::stream_ref to launch the kernel into
|
|
//!
|
|
//! @param __conf
|
|
//! configuration for this launch
|
|
//!
|
|
//! @param __kernel
|
|
//! kernel function to be launched
|
|
//!
|
|
//! @param __args
|
|
//! arguments to be passed into the kernel function
|
|
_CCCL_TEMPLATE(
|
|
typename... _ExpArgs, typename... _ActArgs, typename _Submitter, typename... _Config, typename _Dimensions)
|
|
_CCCL_REQUIRES(work_submitter<_Submitter> _CCCL_AND(sizeof...(_ExpArgs) == sizeof...(_ActArgs)))
|
|
_CCCL_HOST_API auto launch(_Submitter&& __submitter,
|
|
const kernel_config<_Dimensions, _Config...>& __conf,
|
|
void (*__kernel)(_ExpArgs...),
|
|
_ActArgs&&... __args)
|
|
{
|
|
__ensure_current_context __dev_setter{__submitter};
|
|
return ::cuda::__launch_impl<_ExpArgs...>(
|
|
cuda::__forward_or_cast_to_stream_ref<_Submitter>(__submitter), //
|
|
__conf,
|
|
::cuda::__get_cufunction_of(reinterpret_cast<const void*>(__kernel)),
|
|
launch_transform(::cuda::__stream_or_invalid(__submitter), ::cuda::std::forward<_ActArgs>(__args))...);
|
|
}
|
|
|
|
_CCCL_END_NAMESPACE_CUDA
|
|
|
|
# include <cuda/std/__cccl/epilogue.h>
|
|
|
|
#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
|
|
|
|
#endif // _CUDA___LAUNCH_LAUNCH_H
|