[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/libcudacxx/include/cuda/__functional/address_stability.h

@@ -0,0 +1,151 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FUNCTIONAL_ADDRESS_STABILITY_H
+#define _CUDA___FUNCTIONAL_ADDRESS_STABILITY_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__functional/always_true_false.h>
+#include <cuda/__fwd/iterator.h>
+#include <cuda/std/__functional/not_fn.h>
+#include <cuda/std/__functional/operations.h>
+#include <cuda/std/__functional/ranges_operations.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+#include <cuda/std/__type_traits/is_class.h>
+#include <cuda/std/__type_traits/is_enum.h>
+#include <cuda/std/__type_traits/is_void.h>
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+//! Trait telling whether a function object type F does not rely on the memory addresses of its arguments. The nested
+//! value is true when the addresses of the arguments do not matter and arguments can be provided from arbitrary copies
+//! of the respective sources. This trait can be specialized for custom function objects types.
+//! @see proclaim_copyable_arguments
+template <typename F, typename SFINAE = void>
+struct proclaims_copyable_arguments : ::cuda::std::false_type
+{};
+
+template <typename F, typename... Args>
+inline constexpr bool proclaims_copyable_arguments_v = proclaims_copyable_arguments<F, Args...>::value;
+
+// Wrapper for a callable to mark it as permitting copied arguments
+template <typename F>
+struct __callable_permitting_copied_arguments : F
+{
+  using F::operator();
+};
+
+template <typename F>
+struct proclaims_copyable_arguments<__callable_permitting_copied_arguments<F>> : ::cuda::std::true_type
+{};
+
+//! Creates a new function object from an existing one, which is marked as permitting its arguments to be copies of
+//! whatever source they come from. This implies that the addresses of the arguments are irrelevant to the function
+//! object. Some algorithms, like thrust::transform, can benefit from this information and choose a more efficient
+//! implementation.
+//! @see proclaims_copyable_arguments
+template <typename F>
+[[nodiscard]] _CCCL_API constexpr auto proclaim_copyable_arguments(F&& f)
+{
+  if constexpr (proclaims_copyable_arguments<F>::value)
+  { // If F is already marked then we do not need to wrap it
+    return f;
+  }
+  else
+  {
+    return __callable_permitting_copied_arguments<::cuda::std::decay_t<F>>{::cuda::std::forward<F>(f)};
+  }
+}
+
+// Specializations for libcu++ function objects are provided here to not pull this include into `<cuda/std/...>` headers
+
+template <typename _Fn>
+struct proclaims_copyable_arguments<::cuda::std::__not_fn_t<_Fn>> : proclaims_copyable_arguments<_Fn>
+{};
+
+template <typename _Fn>
+struct proclaims_copyable_arguments<zip_function<_Fn>> : proclaims_copyable_arguments<_Fn>
+{};
+
+template <typename _Tp>
+struct __has_builtin_operators
+    : ::cuda::std::bool_constant<!::cuda::std::is_class_v<_Tp> && !::cuda::std::is_enum_v<_Tp>
+                                 && !::cuda::std::is_void_v<_Tp>>
+{};
+
+#define _LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(functor)                                         \
+  /*we know what plus<T> etc. does if T is not a type that could have a weird operatorX() */ \
+  template <typename _Tp>                                                                    \
+  struct proclaims_copyable_arguments<functor<_Tp>> : ::cuda::__has_builtin_operators<_Tp>   \
+  {};                                                                                        \
+  /*we do not know what plus<void> etc. does, which depends on the types it is invoked on */ \
+  template <>                                                                                \
+  struct proclaims_copyable_arguments<functor<void>> : ::cuda::std::false_type               \
+  {};
+
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::plus)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::minus)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::multiplies)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::divides)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::modulus)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::negate)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::bit_and)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::bit_not)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::bit_or)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::bit_xor)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::equal_to)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::not_equal_to)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::less)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::less_equal)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::greater_equal)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::greater)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::logical_and)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::logical_not)
+_LIBCUDACXX_MARK_CAN_COPY_ARGUMENTS(::cuda::std::logical_or)
+
+#define _LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(functor)                                              \
+  /*we do not know what equal_to etc. does, which depends on the types and their operator== it is invoked on */ \
+  template <>                                                                                                   \
+  struct proclaims_copyable_arguments<functor> : ::cuda::std::false_type                                        \
+  {};
+
+_LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(::cuda::std::ranges::equal_to)
+_LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(::cuda::std::ranges::not_equal_to)
+_LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(::cuda::std::ranges::less)
+_LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(::cuda::std::ranges::less_equal)
+_LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(::cuda::std::ranges::greater)
+_LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS(::cuda::std::ranges::greater_equal)
+
+#undef _LIBCUDACXX_MARK_RANGE_FUNCTOR_CAN_COPY_ARGUMENTS
+
+// always_true and always_false never inspect the addresses of their arguments
+template <>
+struct proclaims_copyable_arguments<::cuda::always_true> : ::cuda::std::true_type
+{};
+template <>
+struct proclaims_copyable_arguments<::cuda::always_false> : ::cuda::std::true_type
+{};
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___FUNCTIONAL_ADDRESS_STABILITY_H

cccl_upstream/libcudacxx/include/cuda/__functional/always_true_false.h

@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FUNCTIONAL_ALWAYS_TRUE_FALSE_H
+#define _CUDA___FUNCTIONAL_ALWAYS_TRUE_FALSE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+//! @brief Function object that always returns \c true regardless of the arguments passed.
+struct always_true
+{
+  template <typename... _Ts>
+  [[nodiscard]] _CCCL_API constexpr bool operator()(_Ts&&...) const noexcept
+  {
+    return true;
+  }
+};
+
+//! @brief Function object that always returns \c false regardless of the arguments passed.
+struct always_false
+{
+  template <typename... _Ts>
+  [[nodiscard]] _CCCL_API constexpr bool operator()(_Ts&&...) const noexcept
+  {
+    return false;
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___FUNCTIONAL_ALWAYS_TRUE_FALSE_H

cccl_upstream/libcudacxx/include/cuda/__functional/call_or.h

@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FUNCTIONAL_CALL_OR_H
+#define _CUDA___FUNCTIONAL_CALL_OR_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__tuple_dir/ignore.h>
+#include <cuda/std/__type_traits/is_callable.h>
+#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+//! @brief `__call_or` is an higher-order function that accepts a function, a default
+//! value, and arguments to call the function with. If the function is callable with the
+//! provided arguments, it invokes the function and returns the result. Otherwise, it
+//! returns the default value.
+struct __call_or_t
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  _CCCL_TEMPLATE(class _Fn, class _Fallback, class... _Args)
+  _CCCL_REQUIRES(::cuda::std::__is_callable_v<_Fn, _Args...>)
+  _CCCL_API constexpr auto operator()(_Fn __fn, _Fallback&&, _Args&&... __args) const
+    noexcept(::cuda::std::__is_nothrow_callable_v<_Fn, _Args...>) -> ::cuda::std::__call_result_t<_Fn, _Args...>
+  {
+    return __fn(static_cast<_Args&&>(__args)...);
+  }
+
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class _Fallback, class... _Args>
+  _CCCL_API constexpr auto operator()(::cuda::std::__ignore_t, _Fallback&& __fallback, _Args&&...) const
+    noexcept(::cuda::std::is_nothrow_move_constructible_v<_Fallback>) -> _Fallback
+  {
+    return static_cast<_Fallback&&>(__fallback);
+  }
+};
+
+_CCCL_GLOBAL_CONSTANT auto __call_or = __call_or_t{};
+
+template <class _Fn, class _Fallback, class... _Args>
+using __call_result_or_t _CCCL_NODEBUG_ALIAS = ::cuda::std::__call_result_t<__call_or_t, _Fn, _Fallback, _Args...>;
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___FUNCTIONAL_CALL_OR_H

cccl_upstream/libcudacxx/include/cuda/__functional/equal_to_value.h

@@ -0,0 +1,57 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FUNCTIONAL_EQUAL_TO_VALUE_H
+#define _CUDA___FUNCTIONAL_EQUAL_TO_VALUE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__type_traits/is_comparable.h>
+#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+//! @brief `equal_to_value` is a function object that checks if a value is equal to a stored value.
+//! @tparam _Tp The type of the value to be compared.
+template <typename _Tp>
+struct equal_to_value
+{
+  _Tp __value_;
+
+  _CCCL_API explicit constexpr equal_to_value(const _Tp& __value) noexcept(
+    ::cuda::std::is_nothrow_copy_constructible_v<_Tp>)
+      : __value_(__value)
+  {}
+
+  _CCCL_TEMPLATE(class _Up)
+  _CCCL_REQUIRES(::cuda::std::__is_cpp17_equality_comparable_v<_Tp, _Up>)
+  [[nodiscard]] _CCCL_API constexpr bool operator()(const _Up& __lhs) const
+    noexcept(::cuda::std::__is_cpp17_nothrow_equality_comparable_v<_Tp, _Up>)
+  {
+    return static_cast<bool>(__lhs == __value_);
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___FUNCTIONAL_EQUAL_TO_VALUE_H

cccl_upstream/libcudacxx/include/cuda/__functional/for_each_canceled.h

@@ -0,0 +1,321 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA__FUNCTIONAL_FOR_EACH_CANCELED_H
+#define _CUDA__FUNCTIONAL_FOR_EACH_CANCELED_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#if _CCCL_CUDA_COMPILATION()
+
+#  include <cuda/std/__functional/invoke.h>
+#  include <cuda/std/__utility/move.h>
+#  include <cuda/std/__utility/unreachable.h>
+#  include <cuda/std/cstdint>
+
+#  include <nv/target>
+
+#  include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE
+
+#  if __cccl_ptx_isa >= 870
+
+#    if _CCCL_HAS_INT128()
+using _QueryCancelResult = __uint128_t;
+#    else // ^^^ _CCCL_HAS_INT128() ^^^ / vvv !_CCCL_HAS_INT128() vvv
+struct alignas(16) _QueryCancelResult
+{
+  ::cuda::std::uint64_t __lo_;
+  ::cuda::std::uint64_t __hi_;
+};
+#    endif // ^^^ !_CCCL_HAS_INT128() ^^^
+
+template <int _Index>
+[[nodiscard]] _CCCL_DEVICE_API int __cluster_get_dim(_QueryCancelResult __result) noexcept
+{
+  unsigned __r;
+
+  asm volatile("{\n\t"
+               ".reg .b128 query_result;");
+#    if _CCCL_HAS_INT128()
+  asm volatile("mov.b128 query_result, %0;" : : "q"(__result));
+#    else // ^^^ _CCCL_HAS_INT128() ^^^ / vvv !_CCCL_HAS_INT128() vvv
+  asm volatile("mov.b128 query_result, {%0, %1};" : : "l"(__result.__lo_), "l"(__result.__hi_));
+#    endif // ^^^ !_CCCL_HAS_INT128() ^^^
+
+  if constexpr (_Index == 0)
+  {
+    asm volatile("clusterlaunchcontrol.query_cancel.get_first_ctaid::x.b32.b128 %0, query_result;"
+                 : "=r"(__r)
+                 :
+                 : "memory");
+  }
+  else if constexpr (_Index == 1)
+  {
+    asm volatile("clusterlaunchcontrol.query_cancel.get_first_ctaid::y.b32.b128 %0, query_result;"
+                 : "=r"(__r)
+                 :
+                 : "memory");
+  }
+  else if constexpr (_Index == 2)
+  {
+    asm volatile("clusterlaunchcontrol.query_cancel.get_first_ctaid::z.b32.b128 %0, query_result;"
+                 : "=r"(__r)
+                 :
+                 : "memory");
+  }
+  else
+  {
+    _CCCL_UNREACHABLE();
+  }
+  asm volatile("}");
+  return __r;
+}
+
+//! This API for implementing work-stealing, repeatedly attempts to cancel the launch of a thread block
+//! from the current grid. On success, it invokes the unary function `__uf` before trying again.
+//! On failure, it returns.
+//!
+//! This API does not provide any memory synchronization.
+//! This API does not guarantee that any thread will invoke `__uf` with the next block index until all
+//! invocatons of `__uf` for the prior block index have returned.
+//!
+//! Preconditions:
+//! - All thread block threads shall call this API exactly once.
+//! - Exactly one thread block thread shall call this API with `__is_leader` equals `true`.
+template <int __ThreadBlockRank = 3, typename __UnaryFunction = void>
+_CCCL_DEVICE_API void __for_each_canceled_block_sm100(::dim3 __block_idx, bool __is_leader, __UnaryFunction __uf)
+{
+  __shared__ ::cuda::std::uint64_t __barrier; // TODO: use 2 barriers and 2 results to avoid last sync threads
+  __shared__ _QueryCancelResult __result;
+  bool __phase = false;
+
+  // Initialize barrier and kick-start try_cancel pipeline:
+  if (__is_leader)
+  {
+    auto __leader_mask = ::__activemask();
+    asm volatile(
+      "{\n\t"
+      ".reg .pred p;\n\t"
+      // elect.sync is a workaround for peeling loop (#nvbug-id)
+      "elect.sync _|p, %2;\n\t"
+      "@p mbarrier.init.shared::cta.b64 [%1], 1;\n\t"
+      // `try_cancel` access the mbarrier using generic-proxy, so no cross-proxy fence required here
+      "@p clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [%0], [%1];\n\t"
+      // This arrive does not order prior memory operations and can be relaxed.
+      "@p mbarrier.arrive.expect_tx.relaxed.cta.shared::cta.b64 _, [%1], 16;\n\t"
+      "}"
+      :
+      : "r"((int) ::__cvta_generic_to_shared(&__result)),
+        "r"((int) ::__cvta_generic_to_shared(&__barrier)),
+        "r"(__leader_mask)
+      : "memory");
+  }
+
+  do
+  {
+    ::cuda::std::invoke(__uf, __block_idx);
+    if (__is_leader)
+    {
+      asm volatile(
+        "{\n\t"
+        ".reg .pred p;\n\t"
+        "waitLoop:\n\t\t"
+        "mbarrier.try_wait.parity.relaxed.cta.shared.b64 p, [%0], %1;\n\t\t"
+        "@!p bra waitLoop;\n\t"
+        "}"
+        :
+        : "r"((int) ::__cvta_generic_to_shared(&__barrier)), "r"((unsigned) __phase)
+        : "memory");
+      __phase = !__phase;
+    }
+    ::__syncthreads(); // All threads of prior thread block have "exited".
+    // Note: this syncthreads provides the .acquire.cta fence preventing
+    // the next query operations from being re-ordered above the poll loop.
+    {
+      int __success = 0;
+      asm volatile("{\n\t"
+                   ".reg .pred p;\t\n"
+                   ".reg .b128 query_result;");
+#    if _CCCL_HAS_INT128()
+      asm volatile("mov.b128 query_result, %0;" : : "q"(__result));
+#    else // ^^^ _CCCL_HAS_INT128() ^^^ / vvv !_CCCL_HAS_INT128() vvv
+      asm volatile("mov.b128 query_result, {%0, %1};" : : "l"(__result.__lo_), "l"(__result.__hi_));
+#    endif // ^^^ !_CCCL_HAS_INT128() ^^^
+      asm volatile("clusterlaunchcontrol.query_cancel.is_canceled.pred.b128 p, query_result;\n\t"
+                   "selp.b32 %0, 1, 0, p;\n\t"
+                   "}\n\t"
+                   : "=r"(__success));
+      if (__success != 1)
+      {
+        // Invalidating mbarrier and synchronizing before exiting not
+        // required since each thread block calls this API at most once.
+        break;
+      }
+    }
+
+    // Read new thread block dimensions
+    ::dim3 __b(::cuda::device::__cluster_get_dim<0>(__result), 1, 1);
+    if constexpr (__ThreadBlockRank >= 2)
+    {
+      __b.y = ::cuda::device::__cluster_get_dim<1>(__result);
+    }
+    if constexpr (__ThreadBlockRank == 3)
+    {
+      __b.z = ::cuda::device::__cluster_get_dim<2>(__result);
+    }
+    __block_idx = __b;
+
+    // Wait for all threads to read __result before issuing next async op.
+    // generic->generic synchronization
+    ::__syncthreads();
+    // TODO: only control-warp requires sync, other warps can arrive
+    // TODO: double-buffering results+barrier pairs using phase to avoids this sync
+
+    if (__is_leader)
+    {
+      auto __leader_mask = ::__activemask();
+      asm volatile(
+        "{\n\t"
+        ".reg .pred p;\n\t"
+        // elect.sync is a workaround for peeling loop (#nvbug-id)
+        "elect.sync _|p, %2;\n\t"
+        // generic->async release + acquire synchronization of prior reads:
+        // use bi-directional cross-proxy acq_rel fence instead of uni-dir rel; acq; fences.
+        "@p fence.proxy.async.shared::cta;\n\t"
+        // try to cancel another thread block
+        "@p clusterlaunchcontrol.try_cancel.async.shared::cta.mbarrier::complete_tx::bytes.b128 [%0], [%1];\n\t"
+        "@p mbarrier.arrive.expect_tx.relaxed.cta.shared::cta.b64 _, [%1], 16;\n\t"
+        "}"
+        :
+        : "r"((int) ::__cvta_generic_to_shared(&__result)),
+          "r"((int) ::__cvta_generic_to_shared(&__barrier)),
+          "r"(__leader_mask)
+        : "memory");
+    }
+  } while (true);
+}
+
+#  else // ^^^ __cccl_ptx_isa >= 870 ^^^ / vvv __cccl_ptx_isa < 870 vvv
+template <int __ThreadBlockRank = 3, typename __UnaryFunction = void>
+_CCCL_DEVICE_API void __for_each_canceled_block_sm100(::dim3 __block_idx, bool __is_leader, __UnaryFunction __uf)
+{
+  // We are compiling for SM100 but PTX 8.7 is not supported, so fall back to just calling the function
+  ::cuda::std::invoke(::cuda::std::move(__uf), __block_idx);
+}
+#  endif // ^^^ __cccl_ptx_isa < 870 ^^^
+
+//! This API for implementing work-stealing, repeatedly attempts to cancel the launch of a thread block
+//! from the current grid. On success, it invokes the unary function `__uf` before trying again.
+//! On failure, it returns.
+//!
+//! This API does not provide any memory synchronization.
+//! This API does not guarantee that any thread will invoke `__uf` with the next block index until all
+//! invocatons of `__uf` for the prior block index have returned.
+//!
+//! Preconditions:
+//! - All thread block threads shall call this API exactly once.
+//! - Exactly one thread block thread shall call this API with `__is_leader` equals `true`.
+template <int __ThreadBlockRank = 3, typename __UnaryFunction = void>
+_CCCL_DEVICE_API void __for_each_canceled_block(bool __is_leader, __UnaryFunction __uf)
+{
+  static_assert(__ThreadBlockRank >= 1 && __ThreadBlockRank <= 3, "ThreadBlockRank out-of-range [1, 3].");
+  static_assert(::cuda::std::is_invocable_r_v<void, __UnaryFunction, ::dim3>,
+                "__for_each_canceled_block first argument requires an UnaryFunction with signature: void(dim3).\n"
+                "For example, call with lambda: __for_each_canceled_block([](dim3 block_idx) { ... });");
+  ::dim3 __block_idx = ::dim3(blockIdx.x, 1, 1);
+  if constexpr (__ThreadBlockRank >= 2)
+  {
+    __block_idx = ::dim3(blockIdx.x, blockIdx.y, 1);
+  }
+  if constexpr (__ThreadBlockRank >= 3)
+  {
+    __block_idx = ::dim3(blockIdx.x, blockIdx.y, blockIdx.z);
+  }
+
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_100,
+    (::cuda::device::__for_each_canceled_block_sm100(__block_idx, __is_leader, ::cuda::std::move(__uf));),
+    NV_ANY_TARGET,
+    (::cuda::std::invoke(::cuda::std::move(__uf), __block_idx);))
+}
+
+//! @brief This API used to implement work-stealing, repeatedly attempts to cancel the launch of a thread block
+//!        from the current grid. On success, it invokes the unary function `__uf` before trying again.
+//!        On failure, it returns.
+//!
+//!        This API does not provide any memory synchronization.
+//!        This API does not guarantee that any thread will invoke `__uf` with the next block index until all
+//!        invocatons of `__uf` for the prior block index have returned.
+//!
+//! @pre All thread block threads shall call this API exactly once.
+//! @pre Exactly one thread block thread shall call this API with `__is_leader` equals `true`.
+template <int __ThreadBlockRank = 3, typename __UnaryFunction = void>
+_CCCL_DEVICE_API void for_each_canceled_block(__UnaryFunction __uf)
+{
+  static_assert(__ThreadBlockRank >= 1 && __ThreadBlockRank <= 3,
+                "for_each_canceled_block<ThreadBlockRank>: ThreadBlockRank out-of-range [1, 3].");
+  static_assert(::cuda::std::is_invocable_r_v<void, __UnaryFunction, ::dim3>,
+                "for_each_canceled_block first argument requires an UnaryFunction with signature: void(dim3).\n"
+                "For example, call with lambda: for_each_canceled_block([](dim3 block_idx) { ... });");
+  if constexpr (__ThreadBlockRank == 1)
+  {
+    ::cuda::device::__for_each_canceled_block<1>(threadIdx.x == 0, ::cuda::std::move(__uf));
+  }
+  else if constexpr (__ThreadBlockRank == 2)
+  {
+    ::cuda::device::__for_each_canceled_block<2>(threadIdx.x == 0 && threadIdx.y == 0, ::cuda::std::move(__uf));
+  }
+  else if constexpr (__ThreadBlockRank == 3)
+  {
+    ::cuda::device::__for_each_canceled_block<3>(
+      threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0, ::cuda::std::move(__uf));
+  }
+}
+
+_CCCL_END_NAMESPACE_CUDA_DEVICE
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+//! @brief This API used to implement work-stealing, repeatedly attempts to cancel the launch of a thread block
+//!        from the current grid. On success, it invokes the unary function `__uf` before trying again.
+//!        On failure, it returns.
+//!
+//!        This API does not provide any memory synchronization.
+//!        This API does not guarantee that any thread will invoke `__uf` with the next block index until all
+//!        invocatons of `__uf` for the prior block index have returned.
+//!
+//! @pre All thread block threads shall call this API exactly once.
+//! @pre Exactly one thread block thread shall call this API with `__is_leader` equals `true`.
+//!
+//! @deprecated This function was moved to cuda::device:: namespace.
+template <int __ThreadBlockRank = 3, typename __UnaryFunction = void>
+CCCL_DEPRECATED_BECAUSE("Use cuda::device::for_each_canceled_block instead.") _CCCL_DEVICE_API void
+for_each_canceled_block(__UnaryFunction __uf)
+{
+  ::cuda::device::for_each_canceled_block<__ThreadBlockRank>(::cuda::std::move(__uf));
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#  include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CCCL_CUDA_COMPILATION()
+
+#endif // _CUDA__FUNCTIONAL_FOR_EACH_CANCELED_H

cccl_upstream/libcudacxx/include/cuda/__functional/lazy_call_or.h

@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FUNCTIONAL_LAZY_CALL_OR_H
+#define _CUDA___FUNCTIONAL_LAZY_CALL_OR_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__tuple_dir/ignore.h>
+#include <cuda/std/__type_traits/is_callable.h>
+#include <cuda/std/__utility/forward.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+_CCCL_BEGIN_NAMESPACE_CPO(__lazy_call_or_ns)
+//! @brief `__lazy_call_or` is like `__call_or` except that the fallback value is computed
+//! lazily.
+//!
+//! The fallback value must be a functor that takes no arguments and returns a single
+//! value. The type of the returned fallback value need not be the same as the type of the
+//! computed value.
+struct __fn
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  _CCCL_TEMPLATE(class _Fn, class _FallbackCallable, class... _Args)
+  _CCCL_REQUIRES(::cuda::std::__is_callable_v<_Fn, _Args...>)
+  _CCCL_API constexpr auto operator()(_Fn __fn, _FallbackCallable&&, _Args&&... __args) const
+    noexcept(::cuda::std::__is_nothrow_callable_v<_Fn, _Args...>) -> ::cuda::std::__call_result_t<_Fn, _Args...>
+  {
+    return __fn(::cuda::std::forward<_Args>(__args)...);
+  }
+
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class _FallbackCallable, class... _Args>
+  _CCCL_API constexpr auto operator()(::cuda::std::__ignore_t, _FallbackCallable&& __fallback, _Args&&...) const
+    noexcept(::cuda::std::__is_nothrow_callable_v<_FallbackCallable>) -> ::cuda::std::__call_result_t<_FallbackCallable>
+  {
+    return ::cuda::std::forward<_FallbackCallable>(__fallback)();
+  }
+};
+_CCCL_END_NAMESPACE_CPO
+
+inline namespace __cpo
+{
+_CCCL_GLOBAL_CONSTANT auto __lazy_call_or = __lazy_call_or_ns::__fn{};
+} // namespace __cpo
+
+template <class _Fn, class _FallbackCallable, class... _Args>
+using __lazy_call_result_or_t _CCCL_NODEBUG_ALIAS =
+  ::cuda::std::__call_result_t<__lazy_call_or_ns::__fn, _Fn, _FallbackCallable, _Args...>;
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___FUNCTIONAL_LAZY_CALL_OR_H

cccl_upstream/libcudacxx/include/cuda/__functional/maximum.h

@@ -0,0 +1,77 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_FUNCTIONAL_MAXIMUM_H
+#define _CUDA_FUNCTIONAL_MAXIMUM_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__functional/minimum_maximum_common.h>
+#include <cuda/std/__cmath/min_max.h>
+#include <cuda/std/__type_traits/common_type.h>
+#include <cuda/std/__type_traits/is_extended_floating_point.h>
+#include <cuda/std/__type_traits/is_floating_point.h>
+#include <cuda/std/__utility/ctad_support.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <class _Tp = void>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT maximum
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  [[nodiscard]] _CCCL_API constexpr _Tp operator()(const _Tp& __lhs, const _Tp& __rhs) const
+    noexcept(__is_maximum_minimum_noexcept_v<_Tp, _Tp, _Tp>)
+  {
+    if constexpr (::cuda::std::is_floating_point_v<_Tp> || ::cuda::std::__is_extended_floating_point_v<_Tp>)
+    {
+      return ::cuda::std::fmax(__lhs, __rhs);
+    }
+    else
+    {
+      return (__lhs < __rhs) ? __rhs : __lhs;
+    }
+  }
+};
+_CCCL_CTAD_SUPPORTED_FOR_TYPE(maximum);
+
+template <>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT maximum<void>
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class Tp, class Up, class _Common = ::cuda::std::common_type_t<Tp, Up>>
+  [[nodiscard]] _CCCL_API constexpr _Common operator()(const Tp& __lhs, const Up& __rhs) const
+    noexcept(__is_maximum_minimum_noexcept_v<Tp, Up, _Common>)
+  {
+    if constexpr (::cuda::std::is_floating_point_v<_Common> || ::cuda::std::__is_extended_floating_point_v<_Common>)
+    {
+      return ::cuda::std::fmax(static_cast<_Common>(__lhs), static_cast<_Common>(__rhs));
+    }
+    else
+    {
+      return (__lhs < __rhs) ? __rhs : __lhs;
+    }
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_FUNCTIONAL_MAXIMUM_H

cccl_upstream/libcudacxx/include/cuda/__functional/minimum.h

@@ -0,0 +1,77 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_FUNCTIONAL_MINIMUM_H
+#define _CUDA_FUNCTIONAL_MINIMUM_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__functional/minimum_maximum_common.h>
+#include <cuda/std/__cmath/min_max.h>
+#include <cuda/std/__type_traits/common_type.h>
+#include <cuda/std/__type_traits/is_extended_floating_point.h>
+#include <cuda/std/__type_traits/is_floating_point.h>
+#include <cuda/std/__utility/ctad_support.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <class _Tp = void>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT minimum
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  [[nodiscard]] _CCCL_API constexpr _Tp operator()(const _Tp& __lhs, const _Tp& __rhs) const
+    noexcept(__is_maximum_minimum_noexcept_v<_Tp, _Tp, _Tp>)
+  {
+    if constexpr (::cuda::std::is_floating_point_v<_Tp> || ::cuda::std::__is_extended_floating_point_v<_Tp>)
+    {
+      return ::cuda::std::fmin(__lhs, __rhs);
+    }
+    else
+    {
+      return (__lhs < __rhs) ? __lhs : __rhs;
+    }
+  }
+};
+_CCCL_CTAD_SUPPORTED_FOR_TYPE(minimum);
+
+template <>
+struct _CCCL_TYPE_VISIBILITY_DEFAULT minimum<void>
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class _Tp, class _Up, class _Common = ::cuda::std::common_type_t<_Tp, _Up>>
+  [[nodiscard]] _CCCL_API constexpr _Common operator()(const _Tp& __lhs, const _Up& __rhs) const
+    noexcept(__is_maximum_minimum_noexcept_v<_Tp, _Up, _Common>)
+  {
+    if constexpr (::cuda::std::is_floating_point_v<_Common> || ::cuda::std::__is_extended_floating_point_v<_Common>)
+    {
+      return ::cuda::std::fmin(static_cast<_Common>(__lhs), static_cast<_Common>(__rhs));
+    }
+    else
+    {
+      return (__lhs < __rhs) ? __lhs : __rhs;
+    }
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_FUNCTIONAL_MINIMUM_H

cccl_upstream/libcudacxx/include/cuda/__functional/minimum_maximum_common.h

@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_FUNCTIONAL_MINIMUM_MAXIMUM_COMMON_H
+#define _CUDA_FUNCTIONAL_MINIMUM_MAXIMUM_COMMON_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__floating_point/traits.h>
+#include <cuda/std/__type_traits/common_type.h>
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/is_nothrow_convertible.h>
+#include <cuda/std/__utility/declval.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _Tp, typename _Up, typename _Common = ::cuda::std::common_type_t<_Tp, _Up>, typename _Enable = void>
+constexpr bool __is_maximum_minimum_noexcept_v =
+  noexcept(::cuda::std::declval<_Tp>() < ::cuda::std::declval<_Up>())
+  && ::cuda::std::is_nothrow_convertible_v<_Tp, _Common> && ::cuda::std::is_nothrow_convertible_v<_Up, _Common>;
+
+// Extended floating point types, such as __half and __nv bfloat16 cannot be compared with operator<. We need to
+// handle them separately with SFINAE.
+template <typename _Tp, typename _Up, typename _Common>
+constexpr bool __is_maximum_minimum_noexcept_v<
+  _Tp,
+  _Up,
+  _Common,
+  ::cuda::std::enable_if_t<::cuda::std::__is_ext_nv_fp_v<_Tp> || ::cuda::std::__is_ext_nv_fp_v<_Up>>> = false;
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_FUNCTIONAL_MINIMUM_MAXIMUM_COMMON_H

cccl_upstream/libcudacxx/include/cuda/__functional/operator_properties.h

@@ -0,0 +1,652 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_FUNCTIONAL_OPERATOR_PROPERTIES_H
+#define _CUDA_FUNCTIONAL_OPERATOR_PROPERTIES_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__functional/maximum.h>
+#include <cuda/__functional/minimum.h>
+#include <cuda/__type_traits/is_floating_point.h>
+#include <cuda/std/__floating_point/arithmetic.h>
+#include <cuda/std/__floating_point/constants.h>
+#include <cuda/std/__functional/operations.h>
+#include <cuda/std/__limits/numeric_limits.h>
+#include <cuda/std/__type_traits/always_false.h>
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/is_extended_floating_point.h>
+#include <cuda/std/__type_traits/is_integer.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__type_traits/remove_cv.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+/***********************************************************************************************************************
+ * Associativity
+ **********************************************************************************************************************/
+
+template <class _Op>
+[[nodiscard]] _CCCL_API constexpr bool __is_associative_static_assert()
+{
+  static_assert(::cuda::std::__always_false_v<_Op>,
+                "operator_properties is not specialized for this operator and type combination");
+  return false;
+}
+
+template <class _Op, class _Tp, class Enable = void>
+inline constexpr bool __is_associative_v = __is_associative_static_assert<_Op>();
+
+// strictly speaking, plus (+) and multiply (*) are not associative because of overflow UB
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::std::plus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::std::plus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = false;
+
+template <class _Tp>
+inline constexpr bool __is_associative_v<::cuda::std::plus<>, _Tp> =
+  __is_associative_v<::cuda::std::plus<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool __is_associative_v<::cuda::std::multiplies<_Tp>,
+                                         _Tp,
+                                         ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> = true;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::std::multiplies<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> =
+    false;
+
+template <class _Tp>
+inline constexpr bool __is_associative_v<::cuda::std::multiplies<>, _Tp> =
+  __is_associative_v<::cuda::std::multiplies<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::std::bit_and<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool __is_associative_v<::cuda::std::bit_and<>, _Tp> =
+  __is_associative_v<::cuda::std::bit_and<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::std::bit_or<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool __is_associative_v<::cuda::std::bit_or<>, _Tp> =
+  __is_associative_v<::cuda::std::bit_or<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::std::bit_xor<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool __is_associative_v<::cuda::std::bit_xor<>, _Tp> =
+  __is_associative_v<::cuda::std::bit_xor<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::std::logical_and<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::is_same_v<_Tp, bool>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool __is_associative_v<::cuda::std::logical_and<>, _Tp> =
+  __is_associative_v<::cuda::std::logical_and<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::std::logical_or<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::is_same_v<_Tp, bool>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool __is_associative_v<::cuda::std::logical_or<>, _Tp> =
+  __is_associative_v<::cuda::std::logical_or<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::minimum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::minimum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = true;
+
+template <class _Tp>
+inline constexpr bool __is_associative_v<::cuda::minimum<>, _Tp> = __is_associative_v<::cuda::minimum<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::maximum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::maximum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = true;
+
+template <class _Tp>
+inline constexpr bool __is_associative_v<::cuda::maximum<>, _Tp> = __is_associative_v<::cuda::maximum<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::std::minus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    false;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::std::minus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = false;
+
+template <class _Tp>
+inline constexpr bool __is_associative_v<::cuda::std::minus<>, _Tp> =
+  __is_associative_v<::cuda::std::minus<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::std::divides<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    false;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::std::divides<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> =
+    false;
+
+template <class _Tp>
+inline constexpr bool __is_associative_v<::cuda::std::divides<>, _Tp> =
+  __is_associative_v<::cuda::std::divides<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_associative_v<::cuda::std::modulus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    false;
+
+template <class _Tp>
+inline constexpr bool __is_associative_v<::cuda::std::modulus<>, _Tp> =
+  __is_associative_v<::cuda::std::modulus<_Tp>, _Tp, void>;
+
+template <class _Op, class _Tp>
+inline constexpr bool is_associative_v = __is_associative_v<_Op, ::cuda::std::remove_cv_t<_Tp>>;
+
+/***********************************************************************************************************************
+ * Commutativity
+ **********************************************************************************************************************/
+
+template <class _Op>
+[[nodiscard]] _CCCL_API constexpr bool __is_commutative_static_assert()
+{
+  static_assert(::cuda::std::__always_false_v<_Op>,
+                "operator_properties is not specialized for this operator and type combination");
+  return false;
+}
+
+template <class _Op, class _Tp, class Enable = void>
+inline constexpr bool __is_commutative_v = ::cuda::__is_commutative_static_assert<_Op>();
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::std::plus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::std::plus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = true;
+
+template <class _Tp>
+inline constexpr bool __is_commutative_v<::cuda::std::plus<>, _Tp> =
+  __is_commutative_v<::cuda::std::plus<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool __is_commutative_v<::cuda::std::multiplies<_Tp>,
+                                         _Tp,
+                                         ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> = true;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::std::multiplies<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool __is_commutative_v<::cuda::std::multiplies<>, _Tp> =
+  __is_commutative_v<::cuda::std::multiplies<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::std::bit_and<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool __is_commutative_v<::cuda::std::bit_and<>, _Tp> =
+  __is_commutative_v<::cuda::std::bit_and<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::std::bit_or<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool __is_commutative_v<::cuda::std::bit_or<>, _Tp> =
+  __is_commutative_v<::cuda::std::bit_or<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::std::bit_xor<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool __is_commutative_v<::cuda::std::bit_xor<>, _Tp> =
+  __is_commutative_v<::cuda::std::bit_xor<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::std::logical_and<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::is_same_v<_Tp, bool>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool __is_commutative_v<::cuda::std::logical_and<>, _Tp> =
+  __is_commutative_v<::cuda::std::logical_and<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::std::logical_or<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::is_same_v<_Tp, bool>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool __is_commutative_v<::cuda::std::logical_or<>, _Tp> =
+  __is_commutative_v<::cuda::std::logical_or<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::minimum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::minimum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = true;
+
+template <class _Tp>
+inline constexpr bool __is_commutative_v<::cuda::minimum<>, _Tp> = __is_commutative_v<::cuda::minimum<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::maximum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    true;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::maximum<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = true;
+
+template <class _Tp>
+inline constexpr bool __is_commutative_v<::cuda::maximum<>, _Tp> = __is_commutative_v<::cuda::maximum<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::std::minus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    false;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::std::minus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> = false;
+
+template <class _Tp>
+inline constexpr bool __is_commutative_v<::cuda::std::minus<>, _Tp> =
+  __is_commutative_v<::cuda::std::minus<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::std::divides<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    false;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::std::divides<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::is_floating_point_v<_Tp>>> =
+    false;
+
+template <class _Tp>
+inline constexpr bool __is_commutative_v<::cuda::std::divides<>, _Tp> =
+  __is_commutative_v<::cuda::std::divides<_Tp>, _Tp, void>;
+
+template <class _Tp>
+inline constexpr bool
+  __is_commutative_v<::cuda::std::modulus<_Tp>, _Tp, ::cuda::std::enable_if_t<::cuda::std::__cccl_is_cv_integer_v<_Tp>>> =
+    false;
+
+template <class _Tp>
+inline constexpr bool __is_commutative_v<::cuda::std::modulus<>, _Tp> =
+  __is_commutative_v<::cuda::std::modulus<_Tp>, _Tp, void>;
+
+template <class _Op, class _Tp>
+inline constexpr bool is_commutative_v = __is_commutative_v<_Op, ::cuda::std::remove_cv_t<_Tp>>;
+
+/***********************************************************************************************************************
+ * Internal helpers
+ **********************************************************************************************************************/
+
+template <typename>
+inline constexpr bool __is_cuda_std_plus_v = false;
+
+template <typename... _Tp>
+inline constexpr bool __is_cuda_std_plus_v<::cuda::std::plus<_Tp...>> = true;
+
+template <typename>
+inline constexpr bool __is_cuda_std_multiplies_v = false;
+
+template <typename... _Tp>
+inline constexpr bool __is_cuda_std_multiplies_v<::cuda::std::multiplies<_Tp...>> = true;
+
+template <typename>
+inline constexpr bool __is_cuda_std_bit_and_v = false;
+
+template <typename... _Tp>
+inline constexpr bool __is_cuda_std_bit_and_v<::cuda::std::bit_and<_Tp...>> = true;
+
+template <typename>
+inline constexpr bool __is_cuda_std_bit_or_v = false;
+
+template <typename... _Tp>
+inline constexpr bool __is_cuda_std_bit_or_v<::cuda::std::bit_or<_Tp...>> = true;
+
+template <typename>
+inline constexpr bool __is_cuda_std_bit_xor_v = false;
+
+template <typename... _Tp>
+inline constexpr bool __is_cuda_std_bit_xor_v<::cuda::std::bit_xor<_Tp...>> = true;
+
+template <typename>
+inline constexpr bool __is_cuda_std_logical_and_v = false;
+
+template <typename... _Tp>
+inline constexpr bool __is_cuda_std_logical_and_v<::cuda::std::logical_and<_Tp...>> = true;
+
+template <typename>
+inline constexpr bool __is_cuda_std_logical_or_v = false;
+
+template <typename... _Tp>
+inline constexpr bool __is_cuda_std_logical_or_v<::cuda::std::logical_or<_Tp...>> = true;
+
+template <typename>
+inline constexpr bool __is_cuda_minimum_v = false;
+
+template <typename... _Tp>
+inline constexpr bool __is_cuda_minimum_v<::cuda::minimum<_Tp...>> = true;
+
+template <typename>
+inline constexpr bool __is_cuda_maximum_v = false;
+
+template <typename... _Tp>
+inline constexpr bool __is_cuda_maximum_v<::cuda::maximum<_Tp...>> = true;
+
+/***********************************************************************************************************************
+ * Identity Element
+ **********************************************************************************************************************/
+
+struct __no_identity_element
+{
+  template <class _Tp>
+  [[nodiscard]] _CCCL_API constexpr bool operator==(_Tp&&) noexcept
+  {
+    return false;
+  }
+
+  template <class _Tp>
+  [[nodiscard]] _CCCL_API constexpr bool operator!=(_Tp&&) noexcept
+  {
+    return true;
+  }
+};
+
+template <class _Op, class _Tp>
+[[nodiscard]] _CCCL_API constexpr auto identity_element() noexcept
+{
+  using _Up = ::cuda::std::remove_cv_t<_Tp>;
+  if constexpr (__is_cuda_std_plus_v<_Op>)
+  {
+    if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
+    {
+      return _Up{};
+    }
+    else if constexpr (::cuda::is_floating_point_v<_Up>)
+    {
+      return ::cuda::std::__fp_neg(_Up{}); // -0.0 to preserve negative zero: -0.0 + (-0.0) = -0.0
+    }
+    else
+    {
+      return __no_identity_element{};
+    }
+  }
+  else if constexpr (__is_cuda_std_multiplies_v<_Op>)
+  {
+    if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_floating_point_v<_Up>
+                  || ::cuda::std::is_same_v<_Up, char>)
+    {
+      return _Up{1};
+    }
+    else if constexpr (::cuda::std::__is_extended_floating_point_v<_Up>)
+    {
+      return ::cuda::std::__fp_one<_Up>();
+    }
+    else
+    {
+      return __no_identity_element{};
+    }
+  }
+  else if constexpr (__is_cuda_std_bit_and_v<_Op>)
+  {
+    if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
+    {
+      return static_cast<_Up>(~_Up{});
+    }
+    else
+    {
+      return __no_identity_element{};
+    }
+  }
+  else if constexpr (__is_cuda_std_bit_or_v<_Op> || __is_cuda_std_bit_xor_v<_Op>)
+  {
+    if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
+    {
+      return _Up{};
+    }
+    else
+    {
+      return __no_identity_element{};
+    }
+  }
+  else if constexpr (__is_cuda_std_logical_and_v<_Op>)
+  {
+    if constexpr (::cuda::std::is_same_v<_Up, bool>)
+    {
+      return true;
+    }
+    else
+    {
+      return __no_identity_element{};
+    }
+  }
+  else if constexpr (__is_cuda_std_logical_or_v<_Op>)
+  {
+    if constexpr (::cuda::std::is_same_v<_Up, bool>)
+    {
+      return false;
+    }
+    else
+    {
+      return __no_identity_element{};
+    }
+  }
+  else if constexpr (__is_cuda_minimum_v<_Op>)
+  {
+    if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
+    {
+      return ::cuda::std::numeric_limits<_Up>::max();
+    }
+    else if constexpr (::cuda::is_floating_point_v<_Up>)
+    {
+      return ::cuda::std::numeric_limits<_Up>::infinity();
+    }
+    else
+    {
+      return __no_identity_element{};
+    }
+  }
+  else if constexpr (__is_cuda_maximum_v<_Op>)
+  {
+    if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
+    {
+      return ::cuda::std::numeric_limits<_Up>::lowest();
+    }
+    else if constexpr (::cuda::is_floating_point_v<_Up>)
+    {
+      return ::cuda::std::__fp_neg(::cuda::std::__fp_inf<_Up>());
+    }
+    else
+    {
+      return __no_identity_element{};
+    }
+  }
+  else
+  {
+    return __no_identity_element{};
+  }
+}
+
+template <class _Op, class _Tp, class = void>
+inline constexpr bool has_identity_element_v = false;
+
+template <class _Op, class _Tp>
+inline constexpr bool has_identity_element_v<
+  _Op,
+  _Tp,
+  ::cuda::std::enable_if_t<!::cuda::std::is_same_v<decltype(identity_element<_Op, _Tp>()), __no_identity_element>>> =
+  true;
+
+/***********************************************************************************************************************
+ * Absorbing Element
+ **********************************************************************************************************************/
+
+struct __no_absorbing_element
+{};
+
+template <class _Op, class _Tp>
+[[nodiscard]] _CCCL_API constexpr auto absorbing_element() noexcept
+{
+  using _Up = ::cuda::std::remove_cv_t<_Tp>;
+  if constexpr (__is_cuda_std_multiplies_v<_Op> || __is_cuda_std_bit_and_v<_Op>)
+  {
+    // Multiplication has no absorbing element for floating-point due to NaN, infinity,
+    // and -1.0 * +0.0 = -0.0 (!= +0.0).
+    if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
+    {
+      return _Up{};
+    }
+    else
+    {
+      return __no_absorbing_element{};
+    }
+  }
+  else if constexpr (__is_cuda_std_bit_or_v<_Op>)
+  {
+    if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
+    {
+      return static_cast<_Up>(~_Up{});
+    }
+    else
+    {
+      return __no_absorbing_element{};
+    }
+  }
+  else if constexpr (__is_cuda_std_logical_and_v<_Op>)
+  {
+    if constexpr (::cuda::std::is_same_v<_Up, bool>)
+    {
+      return false;
+    }
+    else
+    {
+      return __no_absorbing_element{};
+    }
+  }
+  else if constexpr (__is_cuda_std_logical_or_v<_Op>)
+  {
+    if constexpr (::cuda::std::is_same_v<_Up, bool>)
+    {
+      return true;
+    }
+    else
+    {
+      return __no_absorbing_element{};
+    }
+  }
+  else if constexpr (__is_cuda_minimum_v<_Op>)
+  {
+    if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
+    {
+      return ::cuda::std::numeric_limits<_Up>::lowest();
+    }
+    else if constexpr (::cuda::is_floating_point_v<_Up>)
+    {
+      return ::cuda::std::__fp_neg(::cuda::std::__fp_inf<_Up>());
+    }
+    else
+    {
+      return __no_absorbing_element{};
+    }
+  }
+  else if constexpr (__is_cuda_maximum_v<_Op>)
+  {
+    if constexpr (::cuda::std::__cccl_is_integer_v<_Up> || ::cuda::std::is_same_v<_Up, char>)
+    {
+      return ::cuda::std::numeric_limits<_Up>::max();
+    }
+    else if constexpr (::cuda::is_floating_point_v<_Up>)
+    {
+      return ::cuda::std::__fp_inf<_Up>();
+    }
+    else
+    {
+      return __no_absorbing_element{};
+    }
+  }
+  else
+  {
+    return __no_absorbing_element{};
+  }
+}
+
+template <class _Op, class _Tp, class = void>
+inline constexpr bool has_absorbing_element_v = false;
+
+template <class _Op, class _Tp>
+inline constexpr bool has_absorbing_element_v<
+  _Op,
+  _Tp,
+  ::cuda::std::enable_if_t<!::cuda::std::is_same_v<decltype(absorbing_element<_Op, _Tp>()), __no_absorbing_element>>> =
+  true;
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_FUNCTIONAL_OPERATOR_PROPERTIES_H

cccl_upstream/libcudacxx/include/cuda/__functional/proclaim_return_type.h

@@ -0,0 +1,109 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___FUNCTIONAL_PROCLAIM_RETURN_TYPE_H
+#define _CUDA___FUNCTIONAL_PROCLAIM_RETURN_TYPE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__functional/invoke.h>
+#include <cuda/std/__type_traits/decay.h>
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__utility/forward.h>
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+namespace __detail
+{
+template <class _Ret, class _DecayFn>
+class __return_type_wrapper
+{
+private:
+  _DecayFn __fn_;
+
+public:
+  __return_type_wrapper() = delete;
+
+  // NOLINTBEGIN(bugprone-forwarding-reference-overload)
+  _CCCL_TEMPLATE(class _Fn)
+  _CCCL_REQUIRES(::cuda::std::is_same_v<::cuda::std::decay_t<_Fn>, _DecayFn>)
+  _CCCL_API constexpr explicit __return_type_wrapper(_Fn&& __fn) noexcept
+      : __fn_(::cuda::std::forward<_Fn>(__fn))
+  {}
+  // NOLINTEND(bugprone-forwarding-reference-overload)
+
+  template <class... _As>
+  _CCCL_API constexpr _Ret operator()(_As&&... __as) & noexcept
+  {
+#if !_CCCL_CUDA_COMPILER(NVCC) || defined(__CUDA_ARCH__)
+    static_assert(::cuda::std::is_same_v<_Ret, ::cuda::std::invoke_result_t<_DecayFn&, _As...>>,
+                  "Return type shall match the proclaimed one exactly");
+#endif // !_CCCL_CUDA_COMPILER(NVCC) || __CUDA_ARCH__
+
+    return ::cuda::std::__invoke(__fn_, ::cuda::std::forward<_As>(__as)...);
+  }
+
+  template <class... _As>
+  _CCCL_API constexpr _Ret operator()(_As&&... __as) && noexcept
+  {
+#if !_CCCL_CUDA_COMPILER(NVCC) || defined(__CUDA_ARCH__)
+    static_assert(::cuda::std::is_same_v<_Ret, ::cuda::std::invoke_result_t<_DecayFn, _As...>>,
+                  "Return type shall match the proclaimed one exactly");
+#endif // !_CCCL_CUDA_COMPILER(NVCC) || __CUDA_ARCH__
+
+    return ::cuda::std::__invoke(::cuda::std::move(__fn_), ::cuda::std::forward<_As>(__as)...);
+  }
+
+  template <class... _As>
+  _CCCL_API constexpr _Ret operator()(_As&&... __as) const& noexcept
+  {
+#if !_CCCL_CUDA_COMPILER(NVCC) || defined(__CUDA_ARCH__)
+    static_assert(::cuda::std::is_same_v<_Ret, ::cuda::std::invoke_result_t<const _DecayFn&, _As...>>,
+                  "Return type shall match the proclaimed one exactly");
+#endif // !_CCCL_CUDA_COMPILER(NVCC) || __CUDA_ARCH__
+
+    return ::cuda::std::__invoke(__fn_, ::cuda::std::forward<_As>(__as)...);
+  }
+
+  template <class... _As>
+  _CCCL_API constexpr _Ret operator()(_As&&... __as) const&& noexcept
+  {
+#if !_CCCL_CUDA_COMPILER(NVCC) || defined(__CUDA_ARCH__)
+    static_assert(::cuda::std::is_same_v<_Ret, ::cuda::std::invoke_result_t<const _DecayFn, _As...>>,
+                  "Return type shall match the proclaimed one exactly");
+#endif // !_CCCL_CUDA_COMPILER(NVCC) || __CUDA_ARCH__
+
+    return ::cuda::std::__invoke(::cuda::std::move(__fn_), ::cuda::std::forward<_As>(__as)...);
+  }
+};
+} // namespace __detail
+
+template <class _Ret, class _Fn>
+_CCCL_API inline __detail::__return_type_wrapper<_Ret, ::cuda::std::decay_t<_Fn>>
+proclaim_return_type(_Fn&& __fn) noexcept
+{
+  return __detail::__return_type_wrapper<_Ret, ::cuda::std::decay_t<_Fn>>(::cuda::std::forward<_Fn>(__fn));
+}
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___FUNCTIONAL_PROCLAIM_RETURN_TYPE_H