[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
@@ -0,0 +1,89 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDA___EXECUTION_DETERMINISM_H
|
||||
#define __CUDA___EXECUTION_DETERMINISM_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__execution/require.h>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__execution/env.h>
|
||||
#include <cuda/std/__type_traits/is_one_of.h>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
namespace determinism
|
||||
{
|
||||
struct __get_determinism_t;
|
||||
|
||||
enum class __determinism_t
|
||||
{
|
||||
__not_guaranteed,
|
||||
__run_to_run,
|
||||
__gpu_to_gpu
|
||||
};
|
||||
|
||||
template <__determinism_t _Guarantee>
|
||||
struct __determinism_holder_t : __requirement
|
||||
{
|
||||
static constexpr __determinism_t value = _Guarantee;
|
||||
|
||||
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto query(const __get_determinism_t&) const noexcept
|
||||
-> __determinism_holder_t<_Guarantee>
|
||||
{
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
using gpu_to_gpu_t = __determinism_holder_t<__determinism_t::__gpu_to_gpu>;
|
||||
using run_to_run_t = __determinism_holder_t<__determinism_t::__run_to_run>;
|
||||
using not_guaranteed_t = __determinism_holder_t<__determinism_t::__not_guaranteed>;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT gpu_to_gpu_t gpu_to_gpu{};
|
||||
_CCCL_GLOBAL_CONSTANT run_to_run_t run_to_run{};
|
||||
_CCCL_GLOBAL_CONSTANT not_guaranteed_t not_guaranteed{};
|
||||
|
||||
struct __get_determinism_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Env)
|
||||
_CCCL_REQUIRES(::cuda::std::execution::__queryable_with<_Env, __get_determinism_t>)
|
||||
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto operator()(const _Env& __env) const noexcept
|
||||
{
|
||||
static_assert(noexcept(__env.query(*this)));
|
||||
return __env.query(*this);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_NODEBUG_API static constexpr auto query(::cuda::std::execution::forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto __get_determinism = __get_determinism_t{};
|
||||
} // namespace determinism
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // __CUDA___EXECUTION_DETERMINISM_H
|
||||
@@ -0,0 +1,94 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDA___EXECUTION_GUARANTEE_H
|
||||
#define __CUDA___EXECUTION_GUARANTEE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__execution/env.h>
|
||||
#include <cuda/std/__type_traits/is_base_of.h>
|
||||
#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
// Base class of all guarantees that can be passed to cuda::execution::guarantee.
|
||||
//
|
||||
// A guarantee is a promise that the caller makes to an algorithm about its input or the problem being solved (e.g. an
|
||||
// upper bound on the total number of items). Algorithms may exploit guarantees to select faster code paths or smaller
|
||||
// intermediate types, but they are never required to. This is the dual of cuda::execution::__requirement, which
|
||||
// describes a property that the caller demands from the algorithm. Unlike requirements, guarantees may be stateful,
|
||||
// i.e. they may carry a runtime value.
|
||||
class __guarantee
|
||||
{};
|
||||
|
||||
// Query used by algorithms to extract the environment of bundled guarantees from an environment assembled by
|
||||
// cuda::execution::guarantee.
|
||||
struct __get_guarantees_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Env)
|
||||
_CCCL_REQUIRES(::cuda::std::execution::__queryable_with<_Env, __get_guarantees_t>)
|
||||
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto operator()(const _Env& __env) const noexcept
|
||||
{
|
||||
static_assert(noexcept(__env.query(*this)), "Guarantees must be queryable without throwing");
|
||||
return __env.query(*this);
|
||||
}
|
||||
|
||||
// Answering the forwarding_query query with true makes __get_guarantees_t a forwarding query: environment adaptors
|
||||
// pass it through to the environments they wrap, so guarantees stay visible to the algorithm no matter how deeply
|
||||
// the environment carrying them is nested.
|
||||
[[nodiscard]]
|
||||
_CCCL_NODEBUG_API static constexpr auto query(::cuda::std::execution::forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto __get_guarantees = __get_guarantees_t{};
|
||||
|
||||
//! @brief Bundles a pack of guarantees into an environment that can be passed to device-wide parallel algorithms.
|
||||
//!
|
||||
//! @param __guarantees The guarantees to bundle. Each guarantee must be nothrow copy constructible. It is stored by
|
||||
//! value, preserving any runtime state it carries.
|
||||
//! @return An environment carrying the bundled guarantees.
|
||||
// The returned property is keyed by __get_guarantees_t so that individual guarantees are only visible to algorithms
|
||||
// through the guarantees environment, mirroring how cuda::execution::require exposes requirements. The key is spelled
|
||||
// __get_guarantees_t{} rather than reusing the __get_guarantees global: odr-using the global inside this inline
|
||||
// host-device function would reference a per-TU internal-linkage object in device passes (see _CCCL_GLOBAL_CONSTANT).
|
||||
template <class... _Guarantees>
|
||||
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto guarantee(_Guarantees... __guarantees) noexcept
|
||||
{
|
||||
static_assert((::cuda::std::is_base_of_v<__guarantee, _Guarantees> && ...),
|
||||
"Only guarantees can be passed to guarantee");
|
||||
static_assert((::cuda::std::is_nothrow_copy_constructible_v<_Guarantees> && ...),
|
||||
"Guarantees must be nothrow copy constructible");
|
||||
|
||||
::cuda::std::execution::env<_Guarantees...> __env{__guarantees...};
|
||||
|
||||
return ::cuda::std::execution::prop{__get_guarantees_t{}, __env};
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // __CUDA___EXECUTION_GUARANTEE_H
|
||||
@@ -0,0 +1,99 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDA___EXECUTION_OUTPUT_ORDERING_H
|
||||
#define __CUDA___EXECUTION_OUTPUT_ORDERING_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__execution/require.h>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__execution/env.h>
|
||||
#include <cuda/std/__type_traits/integral_constant.h>
|
||||
#include <cuda/std/__type_traits/is_one_of.h>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
//! @brief Requirement describing the order in which an algorithm writes its results to the output (e.g. the top-k
|
||||
//! elements of cub::DeviceBatchedTopK).
|
||||
//!
|
||||
//! The available options are:
|
||||
//! - output_ordering::unsorted: the results may be written in any order.
|
||||
//! - output_ordering::sorted: the results are written sorted by key. Among elements that compare
|
||||
//! equal, their relative order is unspecified.
|
||||
//! - output_ordering::stable_sorted: the results are written sorted by key and, among elements that
|
||||
//! compare equal, their relative order matches the order of their source indices (smaller source index first).
|
||||
namespace output_ordering
|
||||
{
|
||||
struct __get_output_ordering_t;
|
||||
|
||||
enum class __output_ordering_t
|
||||
{
|
||||
__sorted,
|
||||
__unsorted,
|
||||
__stable_sorted
|
||||
};
|
||||
|
||||
template <__output_ordering_t _Guarantee>
|
||||
struct _CCCL_DECLSPEC_EMPTY_BASES __output_ordering_holder_t
|
||||
: __requirement
|
||||
, ::cuda::std::integral_constant<__output_ordering_t, _Guarantee>
|
||||
{
|
||||
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto query(const __get_output_ordering_t&) const noexcept
|
||||
-> __output_ordering_holder_t<_Guarantee>
|
||||
{
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
using sorted_t = __output_ordering_holder_t<__output_ordering_t::__sorted>;
|
||||
using stable_sorted_t = __output_ordering_holder_t<__output_ordering_t::__stable_sorted>;
|
||||
using unsorted_t = __output_ordering_holder_t<__output_ordering_t::__unsorted>;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT sorted_t sorted{};
|
||||
_CCCL_GLOBAL_CONSTANT stable_sorted_t stable_sorted{};
|
||||
_CCCL_GLOBAL_CONSTANT unsorted_t unsorted{};
|
||||
|
||||
struct __get_output_ordering_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Env)
|
||||
_CCCL_REQUIRES(::cuda::std::execution::__queryable_with<_Env, __get_output_ordering_t>)
|
||||
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto operator()(const _Env& __env) const noexcept
|
||||
{
|
||||
static_assert(noexcept(__env.query(*this)));
|
||||
return __env.query(*this);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_NODEBUG_API static constexpr auto query(::cuda::std::execution::forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto __get_output_ordering = __get_output_ordering_t{};
|
||||
} // namespace output_ordering
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif
|
||||
43
cccl_upstream/libcudacxx/include/cuda/__execution/policy.h
Normal file
43
cccl_upstream/libcudacxx/include/cuda/__execution/policy.h
Normal file
@@ -0,0 +1,43 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _CUDA___EXECUTION_POLICY_H
|
||||
#define _CUDA___EXECUTION_POLICY_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#if _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
# include <cuda/std/__execution/policy.h>
|
||||
|
||||
# include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
using __cub_parallel_unsequenced_policy =
|
||||
::cuda::std::execution::__execution_policy_base<::cuda::std::execution::__with_backend<
|
||||
static_cast<uint32_t>(::cuda::std::execution::__execution_policy::__parallel_unsequenced),
|
||||
::cuda::std::execution::__execution_backend::__cuda>()>;
|
||||
_CCCL_GLOBAL_CONSTANT __cub_parallel_unsequenced_policy gpu{};
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
# include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // _CCCL_HAS_BACKEND_CUDA()
|
||||
|
||||
#endif // _CUDA___EXECUTION_POLICY_H
|
||||
75
cccl_upstream/libcudacxx/include/cuda/__execution/require.h
Normal file
75
cccl_upstream/libcudacxx/include/cuda/__execution/require.h
Normal file
@@ -0,0 +1,75 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDA___EXECUTION_REQUIRE_H
|
||||
#define __CUDA___EXECUTION_REQUIRE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__execution/env.h>
|
||||
#include <cuda/std/__type_traits/is_base_of.h>
|
||||
#include <cuda/std/__type_traits/is_empty.h>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
class __requirement
|
||||
{};
|
||||
|
||||
struct __get_requirements_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Env)
|
||||
_CCCL_REQUIRES(::cuda::std::execution::__queryable_with<_Env, __get_requirements_t>)
|
||||
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto operator()(const _Env& __env) const noexcept
|
||||
{
|
||||
static_assert(noexcept(__env.query(*this)));
|
||||
return __env.query(*this);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_NODEBUG_API static constexpr auto query(::cuda::std::execution::forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto __get_requirements = __get_requirements_t{};
|
||||
|
||||
template <class... _Requirements>
|
||||
[[nodiscard]] _CCCL_NODEBUG_API auto require(_Requirements...)
|
||||
{
|
||||
static_assert((::cuda::std::is_base_of_v<__requirement, _Requirements> && ...),
|
||||
"Only requirements can be passed to require");
|
||||
static_assert((::cuda::std::is_empty_v<_Requirements> && ...), "Stateful requirements are not implemented");
|
||||
|
||||
// clang < 19 doesn't like this code
|
||||
// since the only requirements we currently allow are in determinism.h and
|
||||
// all of them are stateless, let's ignore incoming parameters
|
||||
::cuda::std::execution::env<_Requirements...> __env{};
|
||||
|
||||
return ::cuda::std::execution::prop{__get_requirements_t{}, __env};
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // __CUDA___EXECUTION_REQUIRE_H
|
||||
@@ -0,0 +1,96 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDA___EXECUTION_TIE_BREAK_H
|
||||
#define __CUDA___EXECUTION_TIE_BREAK_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__execution/require.h>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__execution/env.h>
|
||||
#include <cuda/std/__type_traits/is_one_of.h>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
//! @brief Requirement describing how an algorithm breaks ties among elements that compare equal at its selection
|
||||
//! boundary (e.g. the K-th element of cub::DeviceBatchedTopK).
|
||||
//!
|
||||
//! A tie-break requirement only constrains *which* of the equal-comparing elements end up in the result set; it says
|
||||
//! nothing about the order in which the results are written (that is controlled independently by
|
||||
//! cuda::execution::output_ordering). A tie-break is only meaningful together with a deterministic execution
|
||||
//! requirement (cuda::execution::determinism::run_to_run or gpu_to_gpu).
|
||||
namespace tie_break
|
||||
{
|
||||
struct __get_tie_break_t;
|
||||
|
||||
enum class __tie_break_t
|
||||
{
|
||||
__unspecified, //!< Any (implementation-defined) deterministic tie-break is acceptable.
|
||||
__prefer_smaller_index, //!< Among elements that compare equal, prefer the one(s) with the smaller source index.
|
||||
__prefer_larger_index //!< Among elements that compare equal, prefer the one(s) with the larger source index.
|
||||
};
|
||||
|
||||
template <__tie_break_t _Preference>
|
||||
struct __tie_break_holder_t : __requirement
|
||||
{
|
||||
static constexpr __tie_break_t value = _Preference;
|
||||
|
||||
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto query(const __get_tie_break_t&) const noexcept
|
||||
-> __tie_break_holder_t<_Preference>
|
||||
{
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
using unspecified_t = __tie_break_holder_t<__tie_break_t::__unspecified>;
|
||||
using prefer_smaller_index_t = __tie_break_holder_t<__tie_break_t::__prefer_smaller_index>;
|
||||
using prefer_larger_index_t = __tie_break_holder_t<__tie_break_t::__prefer_larger_index>;
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT unspecified_t unspecified{};
|
||||
_CCCL_GLOBAL_CONSTANT prefer_smaller_index_t prefer_smaller_index{};
|
||||
_CCCL_GLOBAL_CONSTANT prefer_larger_index_t prefer_larger_index{};
|
||||
|
||||
struct __get_tie_break_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Env)
|
||||
_CCCL_REQUIRES(::cuda::std::execution::__queryable_with<_Env, __get_tie_break_t>)
|
||||
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto operator()(const _Env& __env) const noexcept
|
||||
{
|
||||
static_assert(noexcept(__env.query(*this)));
|
||||
return __env.query(*this);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_NODEBUG_API static constexpr auto query(::cuda::std::execution::forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto __get_tie_break = __get_tie_break_t{};
|
||||
} // namespace tie_break
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // __CUDA___EXECUTION_TIE_BREAK_H
|
||||
81
cccl_upstream/libcudacxx/include/cuda/__execution/tune.h
Normal file
81
cccl_upstream/libcudacxx/include/cuda/__execution/tune.h
Normal file
@@ -0,0 +1,81 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of libcu++, the C++ Standard Library for your entire system,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef __CUDA___EXECUTION_TUNE_H
|
||||
#define __CUDA___EXECUTION_TUNE_H
|
||||
|
||||
#include <cuda/std/detail/__config>
|
||||
|
||||
#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
|
||||
# pragma GCC system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
|
||||
# pragma clang system_header
|
||||
#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
|
||||
# pragma system_header
|
||||
#endif // no system header
|
||||
|
||||
#include <cuda/__device/compute_capability.h>
|
||||
#include <cuda/std/__concepts/concept_macros.h>
|
||||
#include <cuda/std/__concepts/semiregular.h>
|
||||
#include <cuda/std/__execution/env.h>
|
||||
#include <cuda/std/__functional/invoke.h>
|
||||
#include <cuda/std/__type_traits/is_empty.h>
|
||||
|
||||
#include <cuda/std/__cccl/prologue.h>
|
||||
|
||||
_CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
struct __get_tuning_t
|
||||
{
|
||||
_CCCL_EXEC_CHECK_DISABLE
|
||||
_CCCL_TEMPLATE(class _Env)
|
||||
_CCCL_REQUIRES(::cuda::std::execution::__queryable_with<_Env, __get_tuning_t>)
|
||||
[[nodiscard]] _CCCL_NODEBUG_API constexpr auto operator()(const _Env& __env) const noexcept
|
||||
{
|
||||
static_assert(noexcept(__env.query(*this)));
|
||||
return __env.query(*this);
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
_CCCL_NODEBUG_API static constexpr auto query(::cuda::std::execution::forwarding_query_t) noexcept -> bool
|
||||
{
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
_CCCL_GLOBAL_CONSTANT auto __get_tuning = __get_tuning_t{};
|
||||
|
||||
//! @rst
|
||||
//! Creates an environment from a pack of policy selectors that can be passed to device-wide parallel algorithms to
|
||||
//! select tunings for different target architectures. See the :ref:`policy selector documentation
|
||||
//! <cub-policy-selectors>` for more information on how algorithms can be tuned.
|
||||
//! @endrst
|
||||
template <class... _PolicySelectors>
|
||||
[[nodiscard]] _CCCL_NODEBUG_API auto tune(_PolicySelectors...)
|
||||
{
|
||||
static_assert((::cuda::std::is_empty_v<_PolicySelectors> && ...), "Policy selectors must be stateless");
|
||||
static_assert((::cuda::std::semiregular<_PolicySelectors> && ...), "Policy selectors must be semiregular types");
|
||||
static_assert((::cuda::std::is_invocable_v<_PolicySelectors, ::cuda::compute_capability> && ...),
|
||||
"Policy selectors must be invocable with cuda::compute_capability");
|
||||
|
||||
// since all the tunings are stateless, let's ignore incoming parameters
|
||||
|
||||
// we use the return type of the policy_selector as tag
|
||||
using tuning_env = ::cuda::std::execution::env<
|
||||
::cuda::std::execution::prop<decltype(_PolicySelectors{}(::cuda::compute_capability{})), _PolicySelectors>...>;
|
||||
|
||||
return ::cuda::std::execution::prop{__get_tuning_t{}, tuning_env{}};
|
||||
}
|
||||
|
||||
_CCCL_END_NAMESPACE_CUDA_EXECUTION
|
||||
|
||||
#include <cuda/std/__cccl/epilogue.h>
|
||||
|
||||
#endif // __CUDA___EXECUTION_TUNE_H
|
||||
Reference in New Issue
Block a user