[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/libcudacxx/include/cuda/std/__numeric/accumulate.h

@@ -0,0 +1,58 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_ACCUMULATE_H
+#define _CUDA_STD___NUMERIC_ACCUMULATE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _Tp>
+[[nodiscard]] _CCCL_API constexpr _Tp accumulate(_InputIterator __first, _InputIterator __last, _Tp __init)
+{
+  for (; __first != __last; ++__first)
+  {
+    __init = ::cuda::std::move(__init) + *__first;
+  }
+  return __init;
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _Tp, class _BinaryOperation>
+[[nodiscard]] _CCCL_API constexpr _Tp
+accumulate(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOperation __binary_op)
+{
+  for (; __first != __last; ++__first)
+  {
+    __init = __binary_op(::cuda::std::move(__init), *__first);
+  }
+  return __init;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_ACCUMULATE_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/adjacent_difference.h

@@ -0,0 +1,74 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_ADJACENT_DIFFERENCE_H
+#define _CUDA_STD___NUMERIC_ADJACENT_DIFFERENCE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__iterator/iterator_traits.h>
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _OutputIterator>
+_CCCL_API constexpr _OutputIterator
+adjacent_difference(_InputIterator __first, _InputIterator __last, _OutputIterator __result)
+{
+  if (__first != __last)
+  {
+    typename iterator_traits<_InputIterator>::value_type __acc(*__first);
+    *__result = __acc;
+    for (++__first, (void) ++__result; __first != __last; ++__first, (void) ++__result)
+    {
+      typename iterator_traits<_InputIterator>::value_type __val(*__first);
+      *__result = __val - ::cuda::std::move(__acc);
+      __acc     = ::cuda::std::move(__val);
+    }
+  }
+  return __result;
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _OutputIterator, class _BinaryOperation>
+_CCCL_API constexpr _OutputIterator adjacent_difference(
+  _InputIterator __first, _InputIterator __last, _OutputIterator __result, _BinaryOperation __binary_op)
+{
+  if (__first != __last)
+  {
+    typename iterator_traits<_InputIterator>::value_type __acc(*__first);
+    *__result = __acc;
+    for (++__first, (void) ++__result; __first != __last; ++__first, (void) ++__result)
+    {
+      typename iterator_traits<_InputIterator>::value_type __val(*__first);
+      *__result = __binary_op(__val, ::cuda::std::move(__acc));
+      __acc     = ::cuda::std::move(__val);
+    }
+  }
+  return __result;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_ADJACENT_DIFFERENCE_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/exclusive_scan.h

@@ -0,0 +1,67 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_EXCLUSIVE_SCAN_H
+#define _CUDA_STD___NUMERIC_EXCLUSIVE_SCAN_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__functional/operations.h>
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp>
+_CCCL_API constexpr _OutputIterator
+exclusive_scan(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _Tp __init, _BinaryOp __b)
+{
+  if (__first != __last)
+  {
+    _Tp __tmp(__b(__init, *__first));
+    while (true)
+    {
+      *__result = ::cuda::std::move(__init);
+      ++__result;
+      ++__first;
+      if (__first == __last)
+      {
+        break;
+      }
+      __init = ::cuda::std::move(__tmp);
+      __tmp  = __b(__init, *__first);
+    }
+  }
+  return __result;
+}
+
+template <class _InputIterator, class _OutputIterator, class _Tp>
+_CCCL_API constexpr _OutputIterator
+exclusive_scan(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _Tp __init)
+{
+  return ::cuda::std::exclusive_scan(__first, __last, __result, __init, ::cuda::std::plus<>());
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_EXCLUSIVE_SCAN_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/gcd_lcm.h

@@ -0,0 +1,78 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_GCD_LCM_H
+#define _CUDA_STD___NUMERIC_GCD_LCM_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__cmath/uabs.h>
+#include <cuda/std/__type_traits/common_type.h>
+#include <cuda/std/__type_traits/is_integral.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__type_traits/is_signed.h>
+#include <cuda/std/__type_traits/make_unsigned.h>
+#include <cuda/std/limits>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+template <class _Tp>
+[[nodiscard]] _CCCL_API constexpr _Tp __gcd(_Tp __m, _Tp __n)
+{
+  static_assert((!is_signed_v<_Tp>) );
+  return __n == 0 ? __m : ::cuda::std::__gcd<_Tp>(__n, __m % __n);
+}
+
+template <class _Tp, class _Up>
+[[nodiscard]] _CCCL_API constexpr common_type_t<_Tp, _Up> gcd(_Tp __m, _Up __n)
+{
+  static_assert((is_integral_v<_Tp> && is_integral_v<_Up>), "Arguments to gcd must be integer types");
+  static_assert((!is_same_v<remove_cv_t<_Tp>, bool>), "First argument to gcd cannot be bool");
+  static_assert((!is_same_v<remove_cv_t<_Up>, bool>), "Second argument to gcd cannot be bool");
+  using _Rp = common_type_t<_Tp, _Up>;
+  using _Wp = make_unsigned_t<_Rp>;
+  return static_cast<_Rp>(::cuda::std::__gcd(static_cast<_Wp>(::cuda::uabs(__m)), static_cast<_Wp>(::cuda::uabs(__n))));
+}
+
+template <class _Tp, class _Up>
+[[nodiscard]] _CCCL_API constexpr common_type_t<_Tp, _Up> lcm(_Tp __m, _Up __n)
+{
+  static_assert((is_integral_v<_Tp> && is_integral_v<_Up>), "Arguments to lcm must be integer types");
+  static_assert((!is_same_v<remove_cv_t<_Tp>, bool>), "First argument to lcm cannot be bool");
+  static_assert((!is_same_v<remove_cv_t<_Up>, bool>), "Second argument to lcm cannot be bool");
+  if (__m == 0 || __n == 0)
+  {
+    return 0;
+  }
+
+  using _Rp         = common_type_t<_Tp, _Up>;
+  using _Wp         = make_unsigned_t<_Rp>;
+  const auto __val1 = ::cuda::uabs(__m) / ::cuda::std::gcd(__m, __n);
+  const auto __val2 = ::cuda::uabs(__n);
+  _CCCL_ASSERT((static_cast<_Wp>(numeric_limits<_Rp>::max()) / __val1 > __val2), "Overflow in lcm");
+  return static_cast<_Rp>(__val1 * __val2);
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_GCD_LCM_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/inclusive_scan.h

@@ -0,0 +1,75 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_INCLUSIVE_SCAN_H
+#define _CUDA_STD___NUMERIC_INCLUSIVE_SCAN_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__functional/operations.h>
+#include <cuda/std/__iterator/iterator_traits.h>
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp>
+_CCCL_API constexpr _OutputIterator
+inclusive_scan(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _BinaryOp __b, _Tp __init)
+{
+  for (; __first != __last; ++__first, (void) ++__result)
+  {
+    __init    = __b(__init, *__first);
+    *__result = __init;
+  }
+  return __result;
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _OutputIterator, class _BinaryOp>
+_CCCL_API constexpr _OutputIterator
+inclusive_scan(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _BinaryOp __b)
+{
+  if (__first != __last)
+  {
+    typename iterator_traits<_InputIterator>::value_type __init = *__first;
+    *__result++                                                 = __init;
+    if (++__first != __last)
+    {
+      return ::cuda::std::inclusive_scan(__first, __last, __result, __b, __init);
+    }
+  }
+
+  return __result;
+}
+
+template <class _InputIterator, class _OutputIterator>
+_CCCL_API constexpr _OutputIterator
+inclusive_scan(_InputIterator __first, _InputIterator __last, _OutputIterator __result)
+{
+  return ::cuda::std::inclusive_scan(__first, __last, __result, ::cuda::std::plus<>());
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_INCLUSIVE_SCAN_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/inner_product.h

@@ -0,0 +1,64 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_INNER_PRODUCT_H
+#define _CUDA_STD___NUMERIC_INNER_PRODUCT_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator1, class _InputIterator2, class _Tp>
+[[nodiscard]] _CCCL_API constexpr _Tp
+inner_product(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _Tp __init)
+{
+  for (; __first1 != __last1; ++__first1, (void) ++__first2)
+  {
+    __init = ::cuda::std::move(__init) + *__first1 * *__first2;
+  }
+  return __init;
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator1, class _InputIterator2, class _Tp, class _BinaryOperation1, class _BinaryOperation2>
+[[nodiscard]] _CCCL_API constexpr _Tp inner_product(
+  _InputIterator1 __first1,
+  _InputIterator1 __last1,
+  _InputIterator2 __first2,
+  _Tp __init,
+  _BinaryOperation1 __binary_op1,
+  _BinaryOperation2 __binary_op2)
+{
+  for (; __first1 != __last1; ++__first1, (void) ++__first2)
+  {
+    __init = __binary_op1(::cuda::std::move(__init), __binary_op2(*__first1, *__first2));
+  }
+  return __init;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_INNER_PRODUCT_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/iota.h

@@ -0,0 +1,43 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_IOTA_H
+#define _CUDA_STD___NUMERIC_IOTA_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _ForwardIterator, class _Tp>
+_CCCL_API constexpr void iota(_ForwardIterator __first, _ForwardIterator __last, _Tp __value_)
+{
+  for (; __first != __last; ++__first, (void) ++__value_)
+  {
+    *__first = __value_;
+  }
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_IOTA_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/midpoint.h

@@ -0,0 +1,93 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_MIDPOINT_H
+#define _CUDA_STD___NUMERIC_MIDPOINT_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/is_floating_point.h>
+#include <cuda/std/__type_traits/is_integral.h>
+#include <cuda/std/__type_traits/is_null_pointer.h>
+#include <cuda/std/__type_traits/is_object.h>
+#include <cuda/std/__type_traits/is_pointer.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__type_traits/is_void.h>
+#include <cuda/std/__type_traits/make_unsigned.h>
+#include <cuda/std/__type_traits/remove_pointer.h>
+#include <cuda/std/cstddef>
+#include <cuda/std/limits>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+template <class _Tp>
+[[nodiscard]]
+_CCCL_API constexpr enable_if_t<is_integral_v<_Tp> && !is_same_v<bool, _Tp> && !is_null_pointer_v<_Tp>, _Tp>
+midpoint(_Tp __a, _Tp __b) noexcept
+{
+  using _Up = make_unsigned_t<_Tp>;
+
+  if (__a > __b)
+  {
+    const _Up __diff = _Up(__a) - _Up(__b);
+    return static_cast<_Tp>(__a - static_cast<_Tp>(__diff / 2));
+  }
+  else
+  {
+    const _Up __diff = _Up(__b) - _Up(__a);
+    return static_cast<_Tp>(__a + static_cast<_Tp>(__diff / 2));
+  }
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+// NOLINTNEXTLINE(bugprone-sizeof-expression)
+template <class _Tp, enable_if_t<is_object_v<_Tp> && !is_void_v<_Tp> && (sizeof(_Tp) > 0), int> = 0>
+[[nodiscard]] _CCCL_API constexpr _Tp* midpoint(_Tp* __a, _Tp* __b) noexcept
+{
+  return __a + ::cuda::std::midpoint(ptrdiff_t(0), __b - __a);
+}
+
+template <typename _Fp>
+[[nodiscard]] _CCCL_API constexpr _Fp __fp_abs(_Fp __f)
+{
+  return __f >= 0 ? __f : -__f;
+}
+
+template <class _Fp>
+[[nodiscard]] _CCCL_API constexpr enable_if_t<is_floating_point_v<_Fp>, _Fp> midpoint(_Fp __a, _Fp __b) noexcept
+{
+  constexpr _Fp __lo = numeric_limits<_Fp>::min() * 2;
+  constexpr _Fp __hi = numeric_limits<_Fp>::max() / 2;
+  return ::cuda::std::__fp_abs(__a) <= __hi && ::cuda::std::__fp_abs(__b) <= __hi
+         ? // typical case: overflow is impossible
+           (__a + __b) / 2
+         : // always correctly rounded
+           ::cuda::std::__fp_abs(__a) < __lo ? __a + __b / 2 : // not safe to halve a
+             ::cuda::std::__fp_abs(__b) < __lo
+             ? __a / 2 + __b
+             : // not safe to halve b
+             __a / 2 + __b / 2; // otherwise correctly rounded
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_MIDPOINT_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/partial_sum.h

@@ -0,0 +1,71 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_PARTIAL_SUM_H
+#define _CUDA_STD___NUMERIC_PARTIAL_SUM_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__iterator/iterator_traits.h>
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _OutputIterator>
+_CCCL_API constexpr _OutputIterator partial_sum(_InputIterator __first, _InputIterator __last, _OutputIterator __result)
+{
+  if (__first != __last)
+  {
+    typename iterator_traits<_InputIterator>::value_type __t(*__first);
+    *__result = __t;
+    for (++__first, (void) ++__result; __first != __last; ++__first, (void) ++__result)
+    {
+      __t       = ::cuda::std::move(__t) + *__first;
+      *__result = __t;
+    }
+  }
+  return __result;
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _OutputIterator, class _BinaryOperation>
+_CCCL_API constexpr _OutputIterator
+partial_sum(_InputIterator __first, _InputIterator __last, _OutputIterator __result, _BinaryOperation __binary_op)
+{
+  if (__first != __last)
+  {
+    typename iterator_traits<_InputIterator>::value_type __t(*__first);
+    *__result = __t;
+    for (++__first, (void) ++__result; __first != __last; ++__first, (void) ++__result)
+    {
+      __t       = __binary_op(::cuda::std::move(__t), *__first);
+      *__result = __t;
+    }
+  }
+  return __result;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_PARTIAL_SUM_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/reduce.h

@@ -0,0 +1,61 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_REDUCE_H
+#define _CUDA_STD___NUMERIC_REDUCE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__functional/operations.h>
+#include <cuda/std/__iterator/iterator_traits.h>
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _Tp, class _BinaryOp>
+[[nodiscard]] _CCCL_API constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOp __b)
+{
+  for (; __first != __last; ++__first)
+  {
+    __init = __b(::cuda::std::move(__init), *__first);
+  }
+  return __init;
+}
+
+template <class _InputIterator, class _Tp>
+[[nodiscard]] _CCCL_API constexpr _Tp reduce(_InputIterator __first, _InputIterator __last, _Tp __init)
+{
+  return ::cuda::std::reduce(__first, __last, __init, ::cuda::std::plus<>());
+}
+
+template <class _InputIterator>
+[[nodiscard]] _CCCL_API constexpr typename iterator_traits<_InputIterator>::value_type
+reduce(_InputIterator __first, _InputIterator __last)
+{
+  return ::cuda::std::reduce(__first, __last, typename iterator_traits<_InputIterator>::value_type{});
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_REDUCE_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/saturating_add.h

@@ -0,0 +1,225 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_SATURATING_ADD_H
+#define _CUDA_STD___NUMERIC_SATURATING_ADD_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__numeric/saturating_add_overflow.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__numeric/saturating_cast.h>
+#include <cuda/std/__type_traits/is_integer.h>
+#include <cuda/std/__type_traits/is_signed.h>
+#include <cuda/std/cstdint>
+
+#include <nv/target>
+
+#if _CCCL_COMPILER(MSVC)
+#  include <intrin.h>
+#endif // _CCCL_COMPILER(MSVC)
+
+#include <cuda/std/__cccl/prologue.h>
+
+#if _CCCL_HAS_BUILTIN(__builtin_elementwise_add_sat)
+#  define _CCCL_BUILTIN_ELEMENTWISE_ADD_SAT(...) __builtin_elementwise_add_sat(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__builtin_elementwise_add_sat)
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+#if !_CCCL_COMPILER(NVRTC)
+template <class _Tp>
+[[nodiscard]] _CCCL_HOST_API _Tp __saturating_add_impl_host(_Tp __x, _Tp __y) noexcept
+{
+#  if defined(_CCCL_BUILTIN_ELEMENTWISE_ADD_SAT)
+// Clang below 21 seems not to be working correctly for 8 and 16-bit types.
+#    if _CCCL_COMPILER(CLANG, <, 21)
+  if constexpr (sizeof(_Tp) < sizeof(int32_t))
+  {
+    return ::cuda::saturating_add_overflow(__x, __y).value;
+  }
+  else
+#    endif // _CCCL_COMPILER(CLANG, <, 21)
+  {
+    return _CCCL_BUILTIN_ELEMENTWISE_ADD_SAT(__x, __y);
+  }
+#  else // ^^^ _CCCL_BUILTIN_ELEMENTWISE_ADD_SAT ^^^ / vvv !_CCCL_BUILTIN_ELEMENTWISE_ADD_SAT vvv
+  if constexpr (is_signed_v<_Tp>)
+  {
+#    if _CCCL_COMPILER(MSVC, >=, 19, 41) && _CCCL_HOST_ARCH(X86_64)
+    if constexpr (sizeof(_Tp) == sizeof(int8_t))
+    {
+      return ::_sat_add_i8(__x, __y);
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(int16_t))
+    {
+      return ::_sat_add_i16(__x, __y);
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(int32_t))
+    {
+      return ::_sat_add_i32(__x, __y);
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(int64_t))
+    {
+      return ::_sat_add_i64(__x, __y);
+    }
+    else
+#    endif // _CCCL_COMPILER(MSVC, >=, 19, 41) && _CCCL_HOST_ARCH(X86_64)
+    {
+      return ::cuda::saturating_add_overflow(__x, __y).value;
+    }
+  }
+  else
+  {
+#    if _CCCL_COMPILER(MSVC, >=, 19, 41) && _CCCL_HOST_ARCH(X86_64)
+    if constexpr (sizeof(_Tp) == sizeof(uint8_t))
+    {
+      return ::_sat_add_u8(__x, __y);
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(uint16_t))
+    {
+      return ::_sat_add_u16(__x, __y);
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(uint32_t))
+    {
+      return ::_sat_add_u32(__x, __y);
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(uint64_t))
+    {
+      return ::_sat_add_u64(__x, __y);
+    }
+    else
+#    endif // _CCCL_COMPILER(MSVC, >=, 19, 41) && _CCCL_HOST_ARCH(X86_64)
+    {
+      return ::cuda::saturating_add_overflow(__x, __y).value;
+    }
+  }
+#  endif // ^^^ !_CCCL_BUILTIN_ELEMENTWISE_ADD_SAT ^^^
+}
+#endif // !_CCCL_COMPILER(NVRTC)
+
+#if _CCCL_CUDA_COMPILATION()
+template <class _Tp>
+[[nodiscard]] _CCCL_DEVICE_API _Tp __saturating_add_impl_device(_Tp __x, _Tp __y) noexcept
+{
+  // Narrow branches differ only when target-specific inline PTX is available.
+  // NOLINTBEGIN(bugprone-branch-clone)
+  if constexpr (is_signed_v<_Tp>)
+  {
+    if constexpr (sizeof(_Tp) == sizeof(int8_t))
+    {
+#  if __cccl_ptx_isa >= 920
+      NV_IF_TARGET(NV_HAS_FEATURE_SM_120f, ({
+                     // Use uint32_t because we want to avoid sign extension.
+                     uint32_t __result;
+                     asm("add.sat.s8x4 %0, %1, %2;"
+                         : "=r"(__result)
+                         : "r"(static_cast<uint32_t>(__x)), "r"(static_cast<uint32_t>(__y)));
+                     return static_cast<_Tp>(__result);
+                   }))
+#  endif // __cccl_ptx_isa >= 920
+      return ::cuda::std::saturating_cast<_Tp>(int32_t{__x} + int32_t{__y});
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(int16_t))
+    {
+#  if __cccl_ptx_isa >= 920
+      NV_IF_TARGET(NV_HAS_FEATURE_SM_120f, ({
+                     // Use uint32_t because we want to avoid sign extension.
+                     uint32_t __result;
+                     asm("add.sat.s16x2 %0, %1, %2;"
+                         : "=r"(__result)
+                         : "r"(static_cast<uint32_t>(__x)), "r"(static_cast<uint32_t>(__y)));
+                     return static_cast<_Tp>(__result);
+                   }))
+#  endif // __cccl_ptx_isa >= 920
+      return ::cuda::std::saturating_cast<_Tp>(int32_t{__x} + int32_t{__y});
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(int32_t))
+    {
+      int32_t __result;
+      asm("add.sat.s32 %0, %1, %2;" : "=r"(__result) : "r"(__x), "r"(__y));
+      return __result;
+    }
+    else
+    {
+      return ::cuda::saturating_add_overflow(__x, __y).value;
+    }
+  }
+  else
+  {
+    if constexpr (sizeof(_Tp) == sizeof(uint8_t))
+    {
+#  if __cccl_ptx_isa >= 920
+      NV_IF_TARGET(NV_HAS_FEATURE_SM_120f, ({
+                     uint32_t __result;
+                     asm("add.sat.u8x4 %0, %1, %2;" : "=r"(__result) : "r"(uint32_t{__x}), "r"(uint32_t{__y}));
+                     return static_cast<_Tp>(__result);
+                   }))
+#  endif // __cccl_ptx_isa >= 920
+      return ::cuda::saturating_add_overflow(__x, __y).value;
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(uint16_t))
+    {
+#  if __cccl_ptx_isa >= 920
+      NV_IF_TARGET(NV_HAS_FEATURE_SM_120f, ({
+                     uint32_t __result;
+                     asm("add.sat.u16x2 %0, %1, %2;" : "=r"(__result) : "r"(uint32_t{__x}), "r"(uint32_t{__y}));
+                     return static_cast<_Tp>(__result);
+                   }))
+#  endif // __cccl_ptx_isa >= 920
+      return ::cuda::saturating_add_overflow(__x, __y).value;
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(uint32_t))
+    {
+#  if __cccl_ptx_isa >= 920
+      NV_IF_TARGET(NV_HAS_FEATURE_SM_120f, ({
+                     uint32_t __result;
+                     asm("add.sat.u32 %0, %1, %2;" : "=r"(__result) : "r"(__x), "r"(__y));
+                     return __result;
+                   }))
+#  endif // __cccl_ptx_isa >= 920
+      return ::cuda::saturating_add_overflow(__x, __y).value;
+    }
+    else
+    {
+      return ::cuda::saturating_add_overflow(__x, __y).value;
+    }
+  }
+  // NOLINTEND(bugprone-branch-clone)
+}
+#endif // _CCCL_CUDA_COMPILATION()
+
+_CCCL_TEMPLATE(class _Tp)
+_CCCL_REQUIRES(__cccl_is_integer_v<_Tp>)
+[[nodiscard]] _CCCL_API constexpr _Tp saturating_add(_Tp __x, _Tp __y) noexcept
+{
+#if !_CCCL_TILE_COMPILATION() // error: asm statement is unsupported in tile code
+  _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+  {
+    NV_IF_ELSE_TARGET(NV_IS_HOST,
+                      (return ::cuda::std::__saturating_add_impl_host(__x, __y);),
+                      (return ::cuda::std::__saturating_add_impl_device(__x, __y);))
+  }
+#endif // !_CCCL_TILE_COMPILATION()
+  return ::cuda::saturating_add_overflow(__x, __y).value;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_SATURATING_ADD_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/saturating_cast.h

@@ -0,0 +1,396 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_SATURATING_CAST_H
+#define _CUDA_STD___NUMERIC_SATURATING_CAST_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__numeric/saturating_overflow_cast.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__type_traits/is_integer.h>
+#include <cuda/std/__type_traits/is_signed.h>
+#include <cuda/std/__type_traits/is_unsigned.h>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+// clang-tidy thinks that branches in __saturating_cast_impl_device are all the same.
+// NOLINTBEGIN(bugprone-branch-clone)
+
+#if _CCCL_CUDA_COMPILATION()
+_CCCL_TEMPLATE(class _To, class _From)
+_CCCL_REQUIRES((sizeof(_To) == sizeof(int8_t)))
+[[nodiscard]] _CCCL_DEVICE_API _To __saturating_cast_impl_device(_From __x, int) noexcept
+{
+  [[maybe_unused]] int __ret;
+
+  if constexpr (sizeof(_From) == sizeof(int8_t))
+  {
+    if constexpr (is_signed_v<_To> && is_unsigned_v<_From>)
+    {
+      asm("cvt.sat.s8.u8 %0, %1;" : "=r"(__ret) : "r"(int{__x}));
+      return static_cast<_To>(__ret);
+    }
+    else if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u8.s8 %0, %1;" : "=r"(__ret) : "r"(int{__x}));
+      return static_cast<_To>(__ret);
+    }
+    else
+    {
+      return __x;
+    }
+  }
+  else if constexpr (sizeof(_From) == sizeof(int16_t))
+  {
+    if constexpr (is_signed_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.s8.s16 %0, %1;" : "=r"(__ret) : "h"(__x));
+      return static_cast<_To>(__ret);
+    }
+    else if constexpr (is_signed_v<_To> && is_unsigned_v<_From>)
+    {
+      asm("cvt.sat.s8.u16 %0, %1;" : "=r"(__ret) : "h"(__x));
+      return static_cast<_To>(__ret);
+    }
+    else if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u8.s16 %0, %1;" : "=r"(__ret) : "h"(__x));
+      return static_cast<_To>(__ret);
+    }
+    else
+    {
+      asm("cvt.sat.u8.u16 %0, %1;" : "=r"(__ret) : "h"(__x));
+      return static_cast<_To>(__ret);
+    }
+  }
+  else if constexpr (sizeof(_From) == sizeof(int32_t))
+  {
+    if constexpr (is_signed_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.s8.s32 %0, %1;" : "=r"(__ret) : "r"(__x));
+      return static_cast<_To>(__ret);
+    }
+    else if constexpr (is_signed_v<_To> && is_unsigned_v<_From>)
+    {
+      asm("cvt.sat.s8.u32 %0, %1;" : "=r"(__ret) : "r"(__x));
+      return static_cast<_To>(__ret);
+    }
+    else if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u8.s32 %0, %1;" : "=r"(__ret) : "r"(__x));
+      return static_cast<_To>(__ret);
+    }
+    else
+    {
+      asm("cvt.sat.u8.u32 %0, %1;" : "=r"(__ret) : "r"(__x));
+      return static_cast<_To>(__ret);
+    }
+  }
+  else if constexpr (sizeof(_From) == sizeof(int64_t))
+  {
+    if constexpr (is_signed_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.s8.s64 %0, %1;" : "=r"(__ret) : "l"(__x));
+      return static_cast<_To>(__ret);
+    }
+    else if constexpr (is_signed_v<_To> && is_unsigned_v<_From>)
+    {
+      asm("cvt.sat.s8.u64 %0, %1;" : "=r"(__ret) : "l"(__x));
+      return static_cast<_To>(__ret);
+    }
+    else if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u8.s64 %0, %1;" : "=r"(__ret) : "l"(__x));
+      return static_cast<_To>(__ret);
+    }
+    else
+    {
+      asm("cvt.sat.u8.u64 %0, %1;" : "=r"(__ret) : "l"(__x));
+      return static_cast<_To>(__ret);
+    }
+  }
+  else
+  {
+    return ::cuda::saturating_overflow_cast<_To>(__x).value;
+  }
+}
+
+_CCCL_TEMPLATE(class _To, class _From)
+_CCCL_REQUIRES((sizeof(_To) == sizeof(int16_t)))
+[[nodiscard]] _CCCL_DEVICE_API _To __saturating_cast_impl_device(_From __x, int) noexcept
+{
+  [[maybe_unused]] _To __ret;
+
+  if constexpr (sizeof(_From) == sizeof(int8_t))
+  {
+    if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u16.s8 %0, %1;" : "=h"(__ret) : "r"(int{__x}));
+      return __ret;
+    }
+    else
+    {
+      return static_cast<_To>(__x);
+    }
+  }
+  else if constexpr (sizeof(_From) == sizeof(int16_t))
+  {
+    if constexpr (is_signed_v<_To> && is_unsigned_v<_From>)
+    {
+      asm("cvt.sat.s16.u16 %0, %1;" : "=h"(__ret) : "h"(__x));
+      return __ret;
+    }
+    else if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u16.s16 %0, %1;" : "=h"(__ret) : "h"(__x));
+      return __ret;
+    }
+    else
+    {
+      return __x;
+    }
+  }
+  else if constexpr (sizeof(_From) == sizeof(int32_t))
+  {
+    if constexpr (is_signed_v<_To> && is_signed_v<_From>)
+    {
+      // There is a bug on Blackwell this PTX instruction giving invalid result for negative inputs. Enable this once
+      // nvbug 6423103 is resolved.
+      NV_IF_ELSE_TARGET(NV_PROVIDES_SM_100, ({ return ::cuda::saturating_overflow_cast<_To>(__x).value; }), ({
+                          asm("cvt.sat.s16.s32 %0, %1;" : "=h"(__ret) : "r"(__x));
+                          return __ret;
+                        }))
+    }
+    else if constexpr (is_signed_v<_To> && is_unsigned_v<_From>)
+    {
+      asm("cvt.sat.s16.u32 %0, %1;" : "=h"(__ret) : "r"(__x));
+      return __ret;
+    }
+    else if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u16.s32 %0, %1;" : "=h"(__ret) : "r"(__x));
+      return __ret;
+    }
+    else
+    {
+      asm("cvt.sat.u16.u32 %0, %1;" : "=h"(__ret) : "r"(__x));
+      return __ret;
+    }
+  }
+  else if constexpr (sizeof(_From) == sizeof(int64_t))
+  {
+    if constexpr (is_signed_v<_To> && is_signed_v<_From>)
+    {
+      // There is a bug on Blackwell this PTX instruction giving invalid result for negative inputs. Enable this once
+      // nvbug 6423103 is resolved.
+      NV_IF_ELSE_TARGET(NV_PROVIDES_SM_100, ({ return ::cuda::saturating_overflow_cast<_To>(__x).value; }), ({
+                          asm("cvt.sat.s16.s64 %0, %1;" : "=h"(__ret) : "l"(__x));
+                          return __ret;
+                        }))
+    }
+    else if constexpr (is_signed_v<_To> && is_unsigned_v<_From>)
+    {
+      asm("cvt.sat.s16.u64 %0, %1;" : "=h"(__ret) : "l"(__x));
+      return __ret;
+    }
+    else if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u16.s64 %0, %1;" : "=h"(__ret) : "l"(__x));
+      return __ret;
+    }
+    else
+    {
+      asm("cvt.sat.u16.u64 %0, %1;" : "=h"(__ret) : "l"(__x));
+      return __ret;
+    }
+  }
+  else
+  {
+    return ::cuda::saturating_overflow_cast<_To>(__x).value;
+  }
+}
+
+_CCCL_TEMPLATE(class _To, class _From)
+_CCCL_REQUIRES((sizeof(_To) == sizeof(int32_t)))
+[[nodiscard]] _CCCL_DEVICE_API _To __saturating_cast_impl_device(_From __x, int) noexcept
+{
+  [[maybe_unused]] _To __ret;
+
+  if constexpr (sizeof(_From) == sizeof(int8_t))
+  {
+    if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u32.s8 %0, %1;" : "=r"(__ret) : "r"(int{__x}));
+      return __ret;
+    }
+    else
+    {
+      return static_cast<_To>(__x);
+    }
+  }
+  else if constexpr (sizeof(_From) == sizeof(int16_t))
+  {
+    if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u32.s16 %0, %1;" : "=r"(__ret) : "h"(__x));
+      return __ret;
+    }
+    else
+    {
+      return static_cast<_To>(__x);
+    }
+  }
+  else if constexpr (sizeof(_From) == sizeof(int32_t))
+  {
+    if constexpr (is_signed_v<_To> && is_unsigned_v<_From>)
+    {
+      asm("cvt.sat.s32.u32 %0, %1;" : "=r"(__ret) : "r"(__x));
+      return __ret;
+    }
+    else if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u32.s32 %0, %1;" : "=r"(__ret) : "r"(__x));
+      return __ret;
+    }
+    else
+    {
+      return __x;
+    }
+  }
+  else if constexpr (sizeof(_From) == sizeof(int64_t))
+  {
+    if constexpr (is_signed_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.s32.s64 %0, %1;" : "=r"(__ret) : "l"(__x));
+      return __ret;
+    }
+    else if constexpr (is_signed_v<_To> && is_unsigned_v<_From>)
+    {
+      asm("cvt.sat.s32.u64 %0, %1;" : "=r"(__ret) : "l"(__x));
+      return __ret;
+    }
+    else if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u32.s64 %0, %1;" : "=r"(__ret) : "l"(__x));
+      return __ret;
+    }
+    else
+    {
+      asm("cvt.sat.u32.u64 %0, %1;" : "=r"(__ret) : "l"(__x));
+      return __ret;
+    }
+  }
+  else
+  {
+    return ::cuda::saturating_overflow_cast<_To>(__x).value;
+  }
+}
+
+_CCCL_TEMPLATE(class _To, class _From)
+_CCCL_REQUIRES((sizeof(_To) == sizeof(int64_t)))
+[[nodiscard]] _CCCL_DEVICE_API _To __saturating_cast_impl_device(_From __x, int) noexcept
+{
+  [[maybe_unused]] _To __ret;
+
+  if constexpr (sizeof(_From) == sizeof(int8_t))
+  {
+    if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u64.s8 %0, %1;" : "=l"(__ret) : "r"(int{__x}));
+      return __ret;
+    }
+    else
+    {
+      return static_cast<_To>(__x);
+    }
+  }
+  else if constexpr (sizeof(_From) == sizeof(int16_t))
+  {
+    if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u64.s16 %0, %1;" : "=l"(__ret) : "h"(__x));
+      return __ret;
+    }
+    else
+    {
+      return static_cast<_To>(__x);
+    }
+  }
+  else if constexpr (sizeof(_From) == sizeof(int32_t))
+  {
+    if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u64.s32 %0, %1;" : "=l"(__ret) : "r"(__x));
+      return __ret;
+    }
+    else
+    {
+      return static_cast<_To>(__x);
+    }
+  }
+  else if constexpr (sizeof(_From) == sizeof(int64_t))
+  {
+    if constexpr (is_signed_v<_To> && is_unsigned_v<_From>)
+    {
+      asm("cvt.sat.s64.u64 %0, %1;" : "=l"(__ret) : "l"(__x));
+      return __ret;
+    }
+    else if constexpr (is_unsigned_v<_To> && is_signed_v<_From>)
+    {
+      asm("cvt.sat.u64.s64 %0, %1;" : "=l"(__ret) : "l"(__x));
+      return __ret;
+    }
+    else
+    {
+      return __x;
+    }
+  }
+  else
+  {
+    return ::cuda::saturating_overflow_cast<_To>(__x).value;
+  }
+}
+
+template <class _To, class _From>
+[[nodiscard]] _CCCL_DEVICE_API _To __saturating_cast_impl_device(_From __x, long) noexcept
+{
+  return ::cuda::saturating_overflow_cast<_To>(__x).value;
+}
+#endif // _CCCL_CUDA_COMPILATION()
+
+// NOLINTEND(bugprone-branch-clone)
+
+_CCCL_TEMPLATE(class _To, class _From)
+_CCCL_REQUIRES(__cccl_is_integer_v<_To> _CCCL_AND __cccl_is_integer_v<_From>)
+[[nodiscard]] _CCCL_API constexpr _To saturating_cast(_From __x) noexcept
+{
+  _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+  {
+    NV_IF_TARGET(NV_IS_DEVICE, ({ return ::cuda::std::__saturating_cast_impl_device<_To>(__x, 0); }))
+  }
+  return ::cuda::saturating_overflow_cast<_To>(__x).value;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_SATURATING_CAST_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/saturating_div.h

@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_SATURATING_DIV_H
+#define _CUDA_STD___NUMERIC_SATURATING_DIV_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__numeric/saturating_div_overflow.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__type_traits/is_integer.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_TEMPLATE(class _Tp)
+_CCCL_REQUIRES(__cccl_is_integer_v<_Tp>)
+[[nodiscard]] _CCCL_API constexpr _Tp saturating_div(_Tp __x, _Tp __y) noexcept
+{
+  return ::cuda::saturating_div_overflow(__x, __y).value;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_SATURATING_DIV_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/saturating_mul.h

@@ -0,0 +1,42 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_SATURATING_MUL_H
+#define _CUDA_STD___NUMERIC_SATURATING_MUL_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__numeric/saturating_mul_overflow.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__type_traits/is_integer.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_TEMPLATE(class _Tp)
+_CCCL_REQUIRES(__cccl_is_integer_v<_Tp>)
+[[nodiscard]] _CCCL_API constexpr _Tp saturating_mul(_Tp __x, _Tp __y) noexcept
+{
+  return ::cuda::saturating_mul_overflow(__x, __y).value;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_SATURATING_MUL_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/saturating_sub.h

@@ -0,0 +1,196 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_SATURATING_SUB_H
+#define _CUDA_STD___NUMERIC_SATURATING_SUB_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__numeric/saturating_sub_overflow.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__numeric/saturating_cast.h>
+#include <cuda/std/__type_traits/is_integer.h>
+#include <cuda/std/__type_traits/is_signed.h>
+#include <cuda/std/cstdint>
+
+#include <nv/target>
+
+#if _CCCL_COMPILER(MSVC)
+#  include <intrin.h>
+#endif // _CCCL_COMPILER(MSVC)
+
+#include <cuda/std/__cccl/prologue.h>
+
+#if _CCCL_HAS_BUILTIN(__builtin_elementwise_sub_sat)
+#  define _CCCL_BUILTIN_ELEMENTWISE_SUB_SAT(...) __builtin_elementwise_sub_sat(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__builtin_elementwise_sub_sat)
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+#if !_CCCL_COMPILER(NVRTC)
+template <class _Tp>
+[[nodiscard]] _CCCL_HOST_API _Tp __saturating_sub_impl_host(_Tp __x, _Tp __y) noexcept
+{
+#  if defined(_CCCL_BUILTIN_ELEMENTWISE_SUB_SAT)
+// Clang below 21 seems not to be working correctly for 8 and 16-bit types.
+#    if _CCCL_COMPILER(CLANG, <, 21)
+  if constexpr (sizeof(_Tp) < sizeof(int32_t))
+  {
+    return ::cuda::saturating_sub_overflow(__x, __y).value;
+  }
+  else
+#    endif // _CCCL_COMPILER(CLANG, <, 21)
+  {
+    return _CCCL_BUILTIN_ELEMENTWISE_SUB_SAT(__x, __y);
+  }
+#  else // ^^^ _CCCL_BUILTIN_ELEMENTWISE_SUB_SAT ^^^ / vvv !_CCCL_BUILTIN_ELEMENTWISE_SUB_SAT vvv
+  if constexpr (is_signed_v<_Tp>)
+  {
+#    if _CCCL_COMPILER(MSVC, >=, 19, 41) && _CCCL_HOST_ARCH(X86_64)
+    if constexpr (sizeof(_Tp) == sizeof(int8_t))
+    {
+      return ::_sat_sub_i8(__x, __y);
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(int16_t))
+    {
+      return ::_sat_sub_i16(__x, __y);
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(int32_t))
+    {
+      return ::_sat_sub_i32(__x, __y);
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(int64_t))
+    {
+      return ::_sat_sub_i64(__x, __y);
+    }
+    else
+#    endif // _CCCL_COMPILER(MSVC, >=, 19, 41) && _CCCL_HOST_ARCH(X86_64)
+    {
+      return ::cuda::saturating_sub_overflow(__x, __y).value;
+    }
+  }
+  else
+  {
+#    if _CCCL_COMPILER(MSVC, >=, 19, 41) && _CCCL_HOST_ARCH(X86_64)
+    if constexpr (sizeof(_Tp) == sizeof(uint8_t))
+    {
+      return ::_sat_sub_u8(__x, __y);
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(uint16_t))
+    {
+      return ::_sat_sub_u16(__x, __y);
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(uint32_t))
+    {
+      return ::_sat_sub_u32(__x, __y);
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(uint64_t))
+    {
+      return ::_sat_sub_u64(__x, __y);
+    }
+    else
+#    endif // _CCCL_COMPILER(MSVC, >=, 19, 41) && _CCCL_HOST_ARCH(X86_64)
+    {
+      return ::cuda::saturating_sub_overflow(__x, __y).value;
+    }
+  }
+#  endif // ^^^ !_CCCL_BUILTIN_ELEMENTWISE_SUB_SAT ^^^
+}
+#endif // !_CCCL_COMPILER(NVRTC)
+
+#if _CCCL_CUDA_COMPILATION()
+template <class _Tp>
+[[nodiscard]] _CCCL_DEVICE_API _Tp __saturating_sub_impl_device(_Tp __x, _Tp __y) noexcept
+{
+  // Narrow branches differ only when target-specific inline PTX is available.
+  // NOLINTBEGIN(bugprone-branch-clone)
+  if constexpr (is_signed_v<_Tp>)
+  {
+    if constexpr (sizeof(_Tp) == sizeof(int8_t))
+    {
+#  if __cccl_ptx_isa >= 920
+      NV_IF_TARGET(NV_HAS_FEATURE_SM_120f, ({
+                     // Use uint32_t because we want to avoid sign extension.
+                     uint32_t __result;
+                     asm("sub.sat.s8x4 %0, %1, %2;"
+                         : "=r"(__result)
+                         : "r"(static_cast<uint32_t>(__x)), "r"(static_cast<uint32_t>(__y)));
+                     return static_cast<_Tp>(__result);
+                   }))
+#  endif // __cccl_ptx_isa >= 920
+      return ::cuda::std::saturating_cast<_Tp>(int32_t{__x} - int32_t{__y});
+    }
+    else if constexpr (sizeof(_Tp) == sizeof(int16_t))
+    {
+      // sub.sat.s16x2 doesn't exist for now
+      return ::cuda::std::saturating_cast<_Tp>(int32_t{__x} - int32_t{__y});
+    }
+    // Disabled due to nvbug 5033045
+    // else if constexpr (sizeof(_Tp) == sizeof(int32_t))
+    // {
+    //   int32_t __result;
+    //   asm volatile("sub.sat.s32 %0, %1, %2;" : "=r"(__result) : "r"(__x), "r"(__y));
+    //   return __result;
+    // }
+    else
+    {
+      return ::cuda::saturating_sub_overflow(__x, __y).value;
+    }
+  }
+  else
+  {
+#  if __cccl_ptx_isa >= 920
+    if constexpr (sizeof(_Tp) == sizeof(uint8_t))
+    {
+      NV_IF_TARGET(NV_HAS_FEATURE_SM_120f, ({
+                     uint32_t __result;
+                     asm("sub.sat.u8x4 %0, %1, %2;" : "=r"(__result) : "r"(uint32_t{__x}), "r"(uint32_t{__y}));
+                     return static_cast<_Tp>(__result);
+                   }))
+      return ::cuda::saturating_sub_overflow(__x, __y).value;
+    }
+    else
+#  endif // __cccl_ptx_isa >= 920
+    {
+      // sub.sat.u16x2 doesn't exist for now
+      return ::cuda::saturating_sub_overflow(__x, __y).value;
+    }
+  }
+  // NOLINTEND(bugprone-branch-clone)
+}
+#endif // _CCCL_CUDA_COMPILATION()
+
+_CCCL_TEMPLATE(class _Tp)
+_CCCL_REQUIRES(__cccl_is_integer_v<_Tp>)
+[[nodiscard]] _CCCL_API constexpr _Tp saturating_sub(_Tp __x, _Tp __y) noexcept
+{
+#if !_CCCL_TILE_COMPILATION() // error: asm statement is unsupported in tile code
+  _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+  {
+    NV_IF_ELSE_TARGET(NV_IS_HOST,
+                      (return ::cuda::std::__saturating_sub_impl_host(__x, __y);),
+                      (return ::cuda::std::__saturating_sub_impl_device(__x, __y);))
+  }
+#endif // !_CCCL_TILE_COMPILATION()
+  return ::cuda::saturating_sub_overflow(__x, __y).value;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_SATURATING_SUB_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/transform_exclusive_scan.h

@@ -0,0 +1,52 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_TRANSFORM_EXCLUSIVE_SCAN_H
+#define _CUDA_STD___NUMERIC_TRANSFORM_EXCLUSIVE_SCAN_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp, class _UnaryOp>
+_CCCL_API constexpr _OutputIterator transform_exclusive_scan(
+  _InputIterator __first, _InputIterator __last, _OutputIterator __result, _Tp __init, _BinaryOp __b, _UnaryOp __u)
+{
+  if (__first != __last)
+  {
+    _Tp __saved = __init;
+    do
+    {
+      __init    = __b(__init, __u(*__first));
+      *__result = __saved;
+      __saved   = __init;
+      ++__result;
+    } while (++__first != __last);
+  }
+  return __result;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_TRANSFORM_EXCLUSIVE_SCAN_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/transform_inclusive_scan.h

@@ -0,0 +1,67 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_TRANSFORM_INCLUSIVE_SCAN_H
+#define _CUDA_STD___NUMERIC_TRANSFORM_INCLUSIVE_SCAN_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__iterator/iterator_traits.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _OutputIterator, class _Tp, class _BinaryOp, class _UnaryOp>
+_CCCL_API constexpr _OutputIterator transform_inclusive_scan(
+  _InputIterator __first, _InputIterator __last, _OutputIterator __result, _BinaryOp __b, _UnaryOp __u, _Tp __init)
+{
+  for (; __first != __last; ++__first, (void) ++__result)
+  {
+    __init    = __b(__init, __u(*__first));
+    *__result = __init;
+  }
+
+  return __result;
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _OutputIterator, class _BinaryOp, class _UnaryOp>
+_CCCL_API constexpr _OutputIterator transform_inclusive_scan(
+  _InputIterator __first, _InputIterator __last, _OutputIterator __result, _BinaryOp __b, _UnaryOp __u)
+{
+  if (__first != __last)
+  {
+    typename iterator_traits<_InputIterator>::value_type __init = __u(*__first);
+    *__result++                                                 = __init;
+    if (++__first != __last)
+    {
+      return ::cuda::std::transform_inclusive_scan(__first, __last, __result, __b, __u, __init);
+    }
+  }
+
+  return __result;
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_TRANSFORM_INCLUSIVE_SCAN_H

cccl_upstream/libcudacxx/include/cuda/std/__numeric/transform_reduce.h

@@ -0,0 +1,74 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_STD___NUMERIC_TRANSFORM_REDUCE_H
+#define _CUDA_STD___NUMERIC_TRANSFORM_REDUCE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__functional/operations.h>
+#include <cuda/std/__iterator/iterator_traits.h>
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator, class _Tp, class _BinaryOp, class _UnaryOp>
+[[nodiscard]] _CCCL_API constexpr _Tp
+transform_reduce(_InputIterator __first, _InputIterator __last, _Tp __init, _BinaryOp __b, _UnaryOp __u)
+{
+  for (; __first != __last; ++__first)
+  {
+    __init = __b(::cuda::std::move(__init), __u(*__first));
+  }
+  return __init;
+}
+
+template <class _InputIterator1, class _InputIterator2, class _Tp, class _BinaryOp1, class _BinaryOp2>
+[[nodiscard]] _CCCL_API constexpr _Tp transform_reduce(
+  _InputIterator1 __first1,
+  _InputIterator1 __last1,
+  _InputIterator2 __first2,
+  _Tp __init,
+  _BinaryOp1 __b1,
+  _BinaryOp2 __b2)
+{
+  for (; __first1 != __last1; ++__first1, (void) ++__first2)
+  {
+    __init = __b1(::cuda::std::move(__init), __b2(*__first1, *__first2));
+  }
+  return __init;
+}
+
+_CCCL_EXEC_CHECK_DISABLE
+template <class _InputIterator1, class _InputIterator2, class _Tp>
+[[nodiscard]] _CCCL_API constexpr _Tp
+transform_reduce(_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _Tp __init)
+{
+  return ::cuda::std::transform_reduce(
+    __first1, __last1, __first2, ::cuda::std::move(__init), ::cuda::std::plus<>(), ::cuda::std::multiplies<>());
+}
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___NUMERIC_TRANSFORM_REDUCE_H