[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/libcudacxx/include/cuda/__memory/address_space.h

@@ -0,0 +1,259 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_ADDRESS_SPACE_H
+#define _CUDA___MEMORY_ADDRESS_SPACE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#if _CCCL_CUDA_COMPILATION()
+
+#  include <cuda/std/__memory/addressof.h>
+#  include <cuda/std/__utility/to_underlying.h>
+
+#  include <nv/target>
+
+#  include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE
+
+//! @brief Address space enumeration for CUDA device code.
+//!
+//!        See https://docs.nvidia.com/cuda/parallel-thread-execution/#state-spaces for more details.
+enum class address_space
+{
+  global, //!< Global state space
+  shared, //!< Shared state space
+  constant, //!< Constant state space
+  local, //!< Local state space
+  grid_constant, //!< Kernel function parameter in the parameter state space
+  cluster_shared, //!< Cluster shared window within the shared state space
+  __max,
+};
+
+[[nodiscard]] _CCCL_DEVICE_API constexpr bool __cccl_is_valid_address_space(address_space __space) noexcept
+{
+  const auto __v = ::cuda::std::to_underlying(__space);
+  return __v >= 0 && __v < ::cuda::std::to_underlying(address_space::__max);
+}
+
+[[nodiscard]] _CCCL_DEVICE_API inline bool __is_smem_valid_ptr(const void* __ptr) noexcept
+{
+  NV_IF_TARGET(NV_PROVIDES_SM_90, (return __ptr != nullptr;), (return true;));
+}
+
+//! @brief Checks if the given pointer is from the specified address state space.
+//! @param __ptr The address to check.
+//! @param __space The address state space to check against.
+//! @return `true` if the pointer is from the specified address space, `false` otherwise.
+[[nodiscard]] _CCCL_DEVICE_API inline bool __internal_is_address_from(const void* __ptr, address_space __space) noexcept
+{
+  _CCCL_ASSERT(::cuda::device::__cccl_is_valid_address_space(__space), "invalid address space");
+  // NVCC and NVRTC < 12.3 have problems tracking the address space of pointers, fallback to inline PTX for them
+  switch (__space)
+  {
+    case address_space::global: {
+#  if _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3)
+      unsigned __ret;
+      asm volatile(
+        "{\n\t"
+        "  .reg .pred p;\n\t"
+        "  isspacep.global p, %1;\n\t"
+        "  selp.u32 %0, 1, 0, p;\n\t"
+        "}\n\t"
+        : "=r"(__ret)
+        : "l"(__ptr));
+      return static_cast<bool>(__ret);
+#  else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ /
+        // vvv !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) vvv
+      bool __p = static_cast<bool>(::__isGlobal(__ptr));
+      if (__p)
+      {
+        _CCCL_ASSUME(__p);
+      }
+      return __p;
+#  endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^
+    }
+    case address_space::constant: {
+#  if _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3)
+      unsigned __ret;
+      asm volatile(
+        "{\n\t"
+        "  .reg .pred p;\n\t"
+        "  isspacep.const p, %1;\n\t"
+        "  selp.u32 %0, 1, 0, p;\n\t"
+        "}\n\t"
+        : "=r"(__ret)
+        : "l"(__ptr));
+      return static_cast<bool>(__ret);
+#  else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ /
+        // vvv !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) vvv
+      bool __p = static_cast<bool>(::__isConstant(__ptr));
+      if (__p)
+      {
+        _CCCL_ASSUME(__p);
+      }
+      return __p;
+#  endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^
+    }
+    case address_space::local: {
+      // __isLocal is buggy until CUDA 13.1, see nvbug 5254298
+#  if _CCCL_CUDA_COMPILER(NVCC, <, 13, 1) || _CCCL_CUDA_COMPILER(NVRTC, <, 13, 1)
+      unsigned __ret;
+      asm volatile(
+        "{\n\t"
+        "  .reg .pred p;\n\t"
+        "  isspacep.local p, %1;\n\t"
+        "  selp.u32 %0, 1, 0, p;\n\t"
+        "}\n\t"
+        : "=r"(__ret)
+        : "l"(__ptr));
+      return static_cast<bool>(__ret);
+#  else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 13, 1) || _CCCL_CUDA_COMPILER(NVRTC, <, 13, 1) ^^^ /
+        // vvv !_CCCL_CUDA_COMPILER(NVCC) && !_CCCL_CUDA_COMPILER(NVRTC) vvv
+      bool __p = static_cast<bool>(::__isLocal(__ptr));
+      if (__p)
+      {
+        _CCCL_ASSUME(__p);
+      }
+      return __p;
+#  endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC) && !_CCCL_CUDA_COMPILER(NVRTC) ^^^
+    }
+    case address_space::grid_constant: {
+#  if _CCCL_CUDA_COMPILER(NVCC, >=, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, >=, 12, 3)
+      NV_IF_ELSE_TARGET(
+        NV_PROVIDES_SM_70,
+        (bool __p = static_cast<bool>(::__isGridConstant(__ptr)); //
+         if (__p) //
+         { //
+           _CCCL_ASSUME(__p); //
+         } //
+         return __p;),
+        (return false;))
+#  else // ^^^ has functional __isGridConstant() ^^^ / vvv no functional __isGridConstant() vvv
+      NV_IF_ELSE_TARGET(
+        NV_PROVIDES_SM_70,
+        (unsigned __ret; //
+         asm volatile("{\n\t"
+                      "  .reg .pred p;\n\t"
+                      "  isspacep.param p, %1;\n\t"
+                      "  selp.u32 %0, 1, 0, p;\n\t"
+                      "}\n\t" : "=r"(__ret) : "l"(__ptr));
+         return static_cast<bool>(__ret);),
+        (return false;))
+#  endif // ^^^ no functional __isGridConstant() ^^^
+    }
+    case address_space::cluster_shared: {
+#  if _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3)
+      NV_IF_ELSE_TARGET(
+        NV_PROVIDES_SM_90,
+        (unsigned __ret; //
+         asm volatile("{\n\t"
+                      "  .reg .pred p;\n\t"
+                      "  isspacep.shared::cluster p, %1;\n\t"
+                      "  selp.u32 %0, 1, 0, p;\n\t"
+                      "}\n\t" : "=r"(__ret) : "l"(__ptr));
+         return static_cast<bool>(__ret);),
+        ([[fallthrough]]; /* to `case shared:` */))
+#  else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ /
+        // vvv !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) vvv
+      NV_IF_ELSE_TARGET(
+        NV_PROVIDES_SM_90,
+        (bool __p = static_cast<bool>(::__isClusterShared(__ptr)); //
+         if (__p) //
+         { //
+           _CCCL_ASSUME(__p); //
+         } //
+         return __p;),
+        ([[fallthrough]]; /* to `case shared:` */))
+#  endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^
+    }
+    case address_space::shared: {
+      // smem can start at address 0x0 before sm_90
+#  if _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3)
+      unsigned __ret;
+      asm volatile(
+        "{\n\t"
+        "  .reg .pred p;\n\t"
+        "  isspacep.shared p, %1;\n\t"
+        "  selp.u32 %0, 1, 0, p;\n\t"
+        "}\n\t"
+        : "=r"(__ret)
+        : "l"(__ptr));
+      return static_cast<bool>(__ret);
+#  else // ^^^ _CCCL_CUDA_COMPILER(NVCC, <, 12, 3) || _CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^ /
+        // vvv !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) vvv
+      bool __p = static_cast<bool>(::__isShared(__ptr));
+      if (__p)
+      {
+        _CCCL_ASSUME(__p);
+      }
+      return __p;
+#  endif // ^^^ !_CCCL_CUDA_COMPILER(NVCC, <, 12, 3) && !_CCCL_CUDA_COMPILER(NVRTC, <, 12, 3) ^^^
+    }
+    default:
+      return false;
+  }
+}
+
+//! @brief Checks if the given pointer is from the specified address state space.
+//! @param __ptr The address to check.
+//! @param __space The address state space to check against.
+//! @return `true` if the pointer is from the specified address space, `false` otherwise.
+[[nodiscard]] _CCCL_DEVICE_API inline bool is_address_from(const void* __ptr, address_space __space) noexcept
+{
+  // The debug assertions intentionally differ but compile out in release builds.
+  // NOLINTBEGIN(bugprone-branch-clone)
+  if (__space == address_space::shared)
+  {
+    _CCCL_ASSERT(::cuda::device::__is_smem_valid_ptr(__ptr), "invalid pointer");
+  }
+  else
+  {
+    _CCCL_ASSERT(__ptr != nullptr, "invalid pointer");
+  }
+  // NOLINTEND(bugprone-branch-clone)
+  return ::cuda::device::__internal_is_address_from(__ptr, __space);
+}
+
+//! @brief Checks if the given pointer is from the specified address state space.
+//! @param __ptr The address to check.
+//! @param __space The address state space to check against.
+//! @return `true` if the pointer is from the specified address space, `false` otherwise.
+[[nodiscard]] _CCCL_DEVICE_API inline bool is_address_from(const volatile void* __ptr, address_space __space) noexcept
+{
+  return ::cuda::device::is_address_from(const_cast<const void*>(__ptr), __space);
+}
+
+//! @brief Checks if the given object is from the specified address state space.
+//! @param __obj The object to check.
+//! @param __space The address state space to check against.
+//! @return `true` if the object is from the specified address space, `false` otherwise.
+template <class _Tp>
+[[nodiscard]] _CCCL_DEVICE_API inline bool is_object_from(_Tp& __obj, address_space __space) noexcept
+{
+  return ::cuda::device::is_address_from(::cuda::std::addressof(__obj), __space);
+}
+
+_CCCL_END_NAMESPACE_CUDA_DEVICE
+
+#  include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CCCL_CUDA_COMPILATION()
+
+#endif // _CUDA___MEMORY_ADDRESS_SPACE_H

cccl_upstream/libcudacxx/include/cuda/__memory/align_down.h

@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_ALIGN_DOWN_H
+#define _CUDA___MEMORY_ALIGN_DOWN_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__memory/is_aligned.h>
+#include <cuda/__memory/is_valid_alignment.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__memory/runtime_assume_aligned.h>
+#include <cuda/std/__type_traits/is_void.h>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__cccl/prologue.h>
+
+#if _CCCL_HAS_BUILTIN(__builtin_align_down)
+#  define _CCCL_BUILTIN_ALIGN_DOWN(...) __builtin_align_down(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__builtin_align_down)
+
+// nvcc doesn't support this builtin in device code, clang-cuda crashes
+#if (_CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG)) && _CCCL_DEVICE_COMPILATION()
+#  undef _CCCL_BUILTIN_ALIGN_DOWN
+#endif // (_CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG)) && _CCCL_DEVICE_COMPILATION()
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _Tp>
+[[nodiscard]] _CCCL_HOST_DEVICE_API _Tp* align_down(_Tp* __ptr, ::cuda::std::size_t __alignment) noexcept
+{
+  using ::cuda::std::uintptr_t;
+  _CCCL_ASSERT(::cuda::__is_valid_alignment<_Tp>(__alignment), "invalid alignment");
+  if constexpr (!::cuda::std::is_void_v<_Tp>)
+  {
+    _CCCL_ASSERT(::cuda::is_aligned(__ptr, alignof(_Tp)), "__ptr is not aligned for _Tp");
+    if (__alignment == alignof(_Tp))
+    {
+      return __ptr;
+    }
+  }
+#if defined(_CCCL_BUILTIN_ALIGN_DOWN)
+  return (_Tp*) _CCCL_BUILTIN_ALIGN_DOWN(__ptr, __alignment);
+#else // ^^^ _CCCL_BUILTIN_ALIGN_DOWN ^^^ / vvv !_CCCL_BUILTIN_ALIGN_DOWN vvv
+  // all code below is translated to a single LOP3.LUT instruction
+  using _Up                = ::cuda::std::remove_cv_t<_Tp>;
+  const auto __char_ptr    = reinterpret_cast<char*>(const_cast<_Up*>(__ptr));
+  const auto __tmp         = static_cast<uintptr_t>(__alignment - 1);
+  const auto __aligned_ptr = reinterpret_cast<char*>( // NOLINT(performance-no-int-to-ptr)
+    reinterpret_cast<uintptr_t>(__ptr) & ~__tmp);
+  // __aligned_ptr and __ptr must be pointers (not values) to apply the optimization
+  // __ptr - (ptr - aligned_ptr) -> __ptr + (aligned_ptr - ptr)
+  const auto __diff = static_cast<::cuda::std::size_t>(__aligned_ptr - __char_ptr);
+  const auto __ret  = reinterpret_cast<_Tp*>(__char_ptr + __diff);
+  return ::cuda::std::__runtime_assume_aligned(__ret, __alignment);
+#endif // ^^^ !_CCCL_BUILTIN_ALIGN_DOWN ^^^
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_ALIGN_DOWN_H

cccl_upstream/libcudacxx/include/cuda/__memory/align_up.h

@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_ALIGN_UP_H
+#define _CUDA___MEMORY_ALIGN_UP_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__memory/is_aligned.h>
+#include <cuda/__memory/is_valid_alignment.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__memory/runtime_assume_aligned.h>
+#include <cuda/std/__type_traits/is_void.h>
+#include <cuda/std/__type_traits/remove_cv.h>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__cccl/prologue.h>
+
+#if _CCCL_HAS_BUILTIN(__builtin_align_up)
+#  define _CCCL_BUILTIN_ALIGN_UP(...) __builtin_align_up(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__builtin_align_up)
+
+// nvcc doesn't support this builtin in device code, clang-cuda crashes
+#if (_CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG)) && _CCCL_DEVICE_COMPILATION()
+#  undef _CCCL_BUILTIN_ALIGN_UP
+#endif // (_CCCL_CUDA_COMPILER(NVCC) || _CCCL_CUDA_COMPILER(CLANG)) && _CCCL_DEVICE_COMPILATION()
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _Tp>
+[[nodiscard]] _CCCL_HOST_DEVICE_API inline _Tp* align_up(_Tp* __ptr, ::cuda::std::size_t __alignment) noexcept
+{
+  using ::cuda::std::uintptr_t;
+  _CCCL_ASSERT(::cuda::__is_valid_alignment<_Tp>(__alignment), "invalid alignment");
+  if constexpr (!::cuda::std::is_void_v<_Tp>)
+  {
+    _CCCL_ASSERT(::cuda::is_aligned(__ptr, alignof(_Tp)), "__ptr is not aligned for _Tp");
+    if (__alignment == alignof(_Tp))
+    {
+      return __ptr;
+    }
+  }
+#if defined(_CCCL_BUILTIN_ALIGN_UP)
+  return (_Tp*) _CCCL_BUILTIN_ALIGN_UP(__ptr, __alignment);
+#else // ^^^ _CCCL_BUILTIN_ALIGN_UP ^^^ / vvv !_CCCL_BUILTIN_ALIGN_UP vvv
+  // all code below is translated to LOP3.LUT + IADD.64 instructions
+  using _Up                = ::cuda::std::remove_cv_t<_Tp>;
+  const auto __char_ptr    = reinterpret_cast<char*>(const_cast<_Up*>(__ptr));
+  const auto __tmp         = static_cast<uintptr_t>(__alignment - 1);
+  const auto __aligned_ptr = reinterpret_cast<char*>( // NOLINT(performance-no-int-to-ptr)
+    (reinterpret_cast<uintptr_t>(__ptr) + __tmp) & ~__tmp);
+  // __aligned_ptr and __ptr must be pointers (not values) to apply the optimization
+  const auto __diff = static_cast<::cuda::std::size_t>(__aligned_ptr - __char_ptr);
+  const auto __ret  = reinterpret_cast<_Tp*>(__char_ptr + __diff);
+  return ::cuda::std::__runtime_assume_aligned(__ret, __alignment);
+#endif // ^^^ !_CCCL_BUILTIN_ALIGN_UP ^^^
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_ALIGN_UP_H

cccl_upstream/libcudacxx/include/cuda/__memory/aligned_size.h

@@ -0,0 +1,61 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_ALIGNED_SIZE_H
+#define _CUDA___MEMORY_ALIGNED_SIZE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__memory/is_valid_alignment.h>
+#include <cuda/std/cstddef>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <::cuda::std::size_t _Alignment>
+struct aligned_size_t
+{
+  static_assert(::cuda::__is_valid_alignment(_Alignment), "invalid alignment");
+
+  static constexpr ::cuda::std::size_t align = _Alignment;
+  ::cuda::std::size_t value;
+
+  _CCCL_API explicit constexpr aligned_size_t(::cuda::std::size_t __s)
+      : value(__s)
+  {
+    _CCCL_ASSERT(value % align == 0,
+                 "aligned_size_t must be constructed with a size that is a multiple of the alignment");
+  }
+  _CCCL_API constexpr operator ::cuda::std::size_t() const
+  {
+    return value;
+  }
+};
+
+template <class, class = void>
+inline constexpr ::cuda::std::size_t __get_size_align_v = 1;
+
+template <class _Tp>
+inline constexpr ::cuda::std::size_t __get_size_align_v<_Tp, ::cuda::std::void_t<decltype(_Tp::align)>> = _Tp::align;
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_ALIGNED_SIZE_H

cccl_upstream/libcudacxx/include/cuda/__memory/check_address.h

@@ -0,0 +1,111 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_IS_VALID_ADDRESS
+#define _CUDA___MEMORY_IS_VALID_ADDRESS
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/cstddef>
+#include <cuda/std/cstdint>
+#if _CCCL_CUDA_COMPILATION()
+#  include <cuda/__memory/address_space.h>
+#  include <cuda/__ptx/instructions/get_sreg.h>
+#endif // _CCCL_CUDA_COMPILATION()
+
+#include <nv/target>
+
+#include <cuda/std/__cccl/prologue.h>
+
+#if _CCCL_CUDA_COMPILATION()
+
+_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE
+
+[[nodiscard]] _CCCL_DEVICE_API inline bool
+__is_smem_valid_address_range(const void* __ptr, ::cuda::std::size_t __n) noexcept
+{
+  if (!::cuda::device::__is_smem_valid_ptr(__ptr))
+  {
+    return false;
+  }
+  if (!::cuda::device::__internal_is_address_from(__ptr, ::cuda::device::address_space::shared))
+  {
+    return false;
+  }
+  // if __ptr is a shared memory pointer, __ptr + __n must also be a valid shared memory pointer
+  if (!::cuda::device::__internal_is_address_from(
+        reinterpret_cast<const char*>(__ptr) + __n, ::cuda::device::address_space::shared))
+  {
+    return false;
+  }
+  return (__n <= ::cuda::ptx::get_sreg_total_smem_size());
+}
+
+_CCCL_END_NAMESPACE_CUDA_DEVICE
+
+#endif // _CCCL_CUDA_COMPILATION()
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+[[nodiscard]] _CCCL_API inline bool __is_valid_address_range(const void* __ptr, ::cuda::std::size_t __n) noexcept
+{
+  if (__n == 0)
+  {
+    return false;
+  }
+
+  // use (~::cuda::std::uintptr_t{0}) instead of cuda::std::numeric_limits<cuda::std::uintptr_t>::max() to avoid
+  // circular dependency because:
+  //    numeric_limits -> bit_cast -> cstring -> check_address
+  // <cuda/std/__utility/cmp.h> also includes cuda/std/limits
+  const auto __limit = (~::cuda::std::uintptr_t{0}) - static_cast<::cuda::std::uintptr_t>(__n);
+
+  if (reinterpret_cast<::cuda::std::uintptr_t>(__ptr) > __limit)
+  {
+    return false;
+  }
+  NV_IF_TARGET(NV_IS_DEVICE, ({
+                 if (::cuda::device::__internal_is_address_from(__ptr, ::cuda::device::address_space::shared)
+                     && !::cuda::device::__is_smem_valid_address_range(__ptr, __n))
+                 {
+                   return false;
+                 }
+               }));
+  return (__ptr != nullptr);
+}
+
+[[nodiscard]] _CCCL_API inline bool __is_valid_address(const void* __ptr) noexcept
+{
+  return ::cuda::__is_valid_address_range(__ptr, 0);
+}
+
+[[nodiscard]] _CCCL_API inline bool
+__are_ptrs_overlapping(const void* __ptr_lhs, const void* __ptr_rhs, ::cuda::std::size_t __n) noexcept
+{
+  const auto __ptr1_start = static_cast<const char*>(__ptr_lhs);
+  const auto __ptr2_start = static_cast<const char*>(__ptr_rhs);
+  const auto __ptr1_end   = __ptr1_start + __n;
+  const auto __ptr2_end   = __ptr2_start + __n;
+  return __ptr1_start < __ptr2_end && __ptr2_start < __ptr1_end;
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_IS_VALID_ADDRESS

cccl_upstream/libcudacxx/include/cuda/__memory/discard_memory.h

@@ -0,0 +1,64 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_DISCARD_MEMORY_H
+#define _CUDA___MEMORY_DISCARD_MEMORY_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__memory/address_space.h>
+#include <cuda/__memory/align_down.h>
+#include <cuda/__memory/align_up.h>
+#include <cuda/std/cstddef>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+_CCCL_HOST_DEVICE_API inline void
+discard_memory([[maybe_unused]] volatile void* __ptr, [[maybe_unused]] ::cuda::std::size_t __nbytes) noexcept {
+// The discard PTX instruction is only available with PTX ISA 7.4 and later
+#if __cccl_ptx_isa >= 740ULL
+  NV_IF_TARGET(NV_PROVIDES_SM_80, ({
+                 _CCCL_ASSERT(__ptr != nullptr, "null pointer passed to discard_memory");
+                 if (!::cuda::device::is_address_from(__ptr, ::cuda::device::address_space::global))
+                 {
+                   return;
+                 }
+
+                 constexpr ::cuda::std::size_t __line_size = 128;
+
+                 // Trim the first block and last block if they're not 128 bytes aligned
+                 const auto __p             = static_cast<char*>(const_cast<void*>(__ptr));
+                 const auto __end_p         = __p + __nbytes;
+                 const auto __start_aligned = ::cuda::align_up(__p, __line_size);
+                 const auto __end_aligned   = ::cuda::align_down(__end_p, __line_size);
+
+                 for (auto __i = __start_aligned; __i < __end_aligned; __i += __line_size)
+                 {
+                   asm volatile("discard.global.L2 [%0], 128;" ::"l"(__i) :);
+                 }
+               }))
+#endif // __cccl_ptx_isa >= 740ULL
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_DISCARD_MEMORY_H

cccl_upstream/libcudacxx/include/cuda/__memory/get_device_address.h

@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_GET_DEVICE_ADDRESS_H
+#define _CUDA___MEMORY_GET_DEVICE_ADDRESS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#if _CCCL_HAS_CTK()
+
+#  include <cuda/__device/device_ref.h>
+#  include <cuda/__runtime/api_wrapper.h>
+#  include <cuda/__runtime/ensure_current_context.h>
+#  include <cuda/std/__memory/addressof.h>
+
+#  include <nv/target>
+
+#  include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+//! @brief Returns the device address of the passed \c __device_object
+//! @param __device_object the object residing in device memory
+//! @warning The user must ensure that the current device is properly set to the device the object was allocated on.
+//! @return Valid pointer to the device object
+template <class _Tp>
+[[nodiscard]] _CCCL_API inline _Tp* get_device_address(_Tp& __device_object)
+{
+  NV_IF_ELSE_TARGET(NV_IS_DEVICE, (return ::cuda::std::addressof(__device_object);), ({
+                      void* __device_ptr = nullptr; //
+                      _CCCL_TRY_CUDA_API(::cudaGetSymbolAddress,
+                                         "failed to call cudaGetSymbolAddress in cuda::get_device_address",
+                                         &__device_ptr,
+                                         __device_object);
+                      return static_cast<_Tp*>(__device_ptr);
+                    }))
+}
+
+#  if !_CCCL_COMPILER(NVRTC)
+//! @brief Returns the address of the passed \c __device_object for the passed \c __device.
+//!
+//! @param __device_object The object residing in device memory.
+//! @param __device The device to query the address for.
+//!
+//! @return Valid pointer to the device object.
+//!
+//! @throws cuda::cuda_error if the operation fails.
+template <class _Tp>
+[[nodiscard]] _CCCL_HOST_API inline _Tp* get_device_address(_Tp& __device_object, device_ref __device)
+{
+  __ensure_current_context __ctx{__device};
+  void* __device_ptr{};
+  _CCCL_TRY_CUDA_API(::cudaGetSymbolAddress,
+                     "failed to call cudaGetSymbolAddress in cuda::get_device_address",
+                     &__device_ptr,
+                     __device_object);
+  return static_cast<_Tp*>(__device_ptr);
+}
+#  endif // !_CCCL_COMPILER(NVRTC)
+
+_CCCL_END_NAMESPACE_CUDA
+
+#  include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CCCL_HAS_CTK()
+
+#endif // _CUDA___MEMORY_GET_DEVICE_ADDRESS_H

cccl_upstream/libcudacxx/include/cuda/__memory/is_aligned.h

@@ -0,0 +1,60 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_IS_ALIGNED_H
+#define _CUDA___MEMORY_IS_ALIGNED_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__memory/is_valid_alignment.h>
+#include <cuda/std/cstddef>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__cccl/prologue.h>
+
+#if _CCCL_HAS_BUILTIN(__builtin_is_aligned)
+#  define _CCCL_BUILTIN_IS_ALIGNED(...) __builtin_is_aligned(__VA_ARGS__)
+#endif // _CCCL_HAS_BUILTIN(__builtin_is_aligned)
+
+// nvcc doesn't support this builtin in device code, clang-cuda crashes
+#if _CCCL_CUDA_COMPILER(NVCC) && _CCCL_DEVICE_COMPILATION()
+#  undef _CCCL_BUILTIN_IS_ALIGNED
+#endif // _CCCL_CUDA_COMPILER(NVCC) && _CCCL_DEVICE_COMPILATION()
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+[[nodiscard]] _CCCL_API inline bool is_aligned(const void* __ptr, ::cuda::std::size_t __alignment) noexcept
+{
+  _CCCL_ASSERT(::cuda::__is_valid_alignment(__alignment), "invalid alignment");
+#if defined(_CCCL_BUILTIN_IS_ALIGNED)
+  return _CCCL_BUILTIN_IS_ALIGNED(__ptr, __alignment);
+#else // ^^^ _CCCL_BUILTIN_IS_ALIGNED ^^^ / vvv !_CCCL_BUILTIN_IS_ALIGNED vvv
+  return (reinterpret_cast<::cuda::std::uintptr_t>(__ptr) & (__alignment - 1)) == 0;
+#endif // ^^^ !_CCCL_BUILTIN_IS_ALIGNED ^^^
+}
+
+[[nodiscard]] _CCCL_API inline bool is_aligned(const volatile void* __ptr, ::cuda::std::size_t __alignment) noexcept
+{
+  return ::cuda::is_aligned(const_cast<const void*>(__ptr), __alignment);
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_IS_ALIGN_H

cccl_upstream/libcudacxx/include/cuda/__memory/is_pointer_accessible.h

@@ -0,0 +1,261 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_IS_POINTER_ACCESSIBLE_H
+#define _CUDA___MEMORY_IS_POINTER_ACCESSIBLE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__device/device_ref.h>
+#include <cuda/__driver/driver_api.h>
+#include <cuda/__runtime/ensure_current_context.h>
+#include <cuda/std/__exception/cuda_error.h>
+#include <cuda/std/__exception/exception_macros.h>
+#include <cuda/std/__type_traits/always_false.h>
+#include <cuda/std/__type_traits/integral_constant.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_DIAG_PUSH
+_CCCL_DIAG_SUPPRESS_CLANG("-Wmissing-braces")
+// clang complains about missing braces in CUmemLocation constructor but GCC complains if we add them
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
+
+#  define _CCCL_THROW_OR_RETURN(_STATUS, _MSG)                                               \
+    if ((_STATUS) != ::cudaSuccess)                                                          \
+    {                                                                                        \
+      if constexpr (_IsNothrow)                                                              \
+      {                                                                                      \
+        return false;                                                                        \
+      }                                                                                      \
+      else                                                                                   \
+      {                                                                                      \
+        _CCCL_THROW(::cuda::cuda_error, (_STATUS), (_MSG), _CCCL_BUILTIN_PRETTY_FUNCTION()); \
+      }                                                                                      \
+    }
+
+template <bool _IsNothrow>
+[[nodiscard]]
+_CCCL_HOST_API inline bool __is_managed(const void* __p, ::cuda::std::bool_constant<_IsNothrow>) noexcept(_IsNothrow)
+{
+  if (__p == nullptr)
+  {
+    return false;
+  }
+  bool __is_managed{};
+  const auto __status =
+    ::cuda::__driver::__pointerGetAttributeNoThrow<::CU_POINTER_ATTRIBUTE_IS_MANAGED>(__is_managed, __p);
+  switch (__status)
+  {
+    case ::cudaSuccess:
+      return __is_managed;
+    case ::cudaErrorInvalidValue:
+      return false;
+    default:
+      _CCCL_THROW_OR_RETURN(__status, "is_managed() failed");
+      return false;
+  }
+}
+
+/**
+ * @brief Checks if a pointer is a managed pointer.
+ *
+ * @param __p The pointer to check.
+ * @return `true` if the pointer is a managed pointer, `false` otherwise.
+ */
+[[nodiscard]]
+_CCCL_HOST_API inline bool is_managed(const void* __p)
+{
+  return ::cuda::__is_managed(__p, ::cuda::std::false_type{});
+}
+
+[[nodiscard]]
+_CCCL_HOST_API inline bool __is_managed_nothrow(const void* __p) noexcept
+{
+  return ::cuda::__is_managed(__p, ::cuda::std::true_type{});
+}
+
+template <bool _IsNothrow>
+[[nodiscard]]
+_CCCL_HOST_API inline bool
+__is_host_accessible(const void* __p, ::cuda::std::bool_constant<_IsNothrow>) noexcept(_IsNothrow)
+{
+  if (__p == nullptr)
+  {
+    return false;
+  }
+  ::CUpointer_attribute __attrs[3] = {
+    ::CU_POINTER_ATTRIBUTE_MEMORY_TYPE, ::CU_POINTER_ATTRIBUTE_IS_MANAGED, ::CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE};
+  auto __memory_type       = static_cast<::CUmemorytype>(0);
+  int __is_managed         = 0;
+  ::CUmemoryPool __mempool = nullptr;
+  void* __results[3]       = {&__memory_type, &__is_managed, &__mempool};
+  const auto __status      = ::cuda::__driver::__pointerGetAttributesNoThrow(__attrs, __results, __p);
+  _CCCL_THROW_OR_RETURN(__status, "Failed to get attributes of a pointer");
+  // (1) check if the pointer is unregistered
+  if (__memory_type == static_cast<::CUmemorytype>(0)
+      || (__mempool == nullptr && (__is_managed || __memory_type == ::CU_MEMORYTYPE_HOST)))
+  {
+    return true;
+  }
+  // (2) check if a memory pool is associated with the pointer
+#  if _CCCL_CTK_AT_LEAST(12, 2)
+  if (__mempool != nullptr)
+  {
+    ::CUmemLocation __prop{::CU_MEM_LOCATION_TYPE_HOST, 0};
+    ::CUmemAccess_flags __pool_flags;
+    const auto __status2 = ::cuda::__driver::__mempoolGetAccessNoThrow(__pool_flags, __mempool, &__prop);
+    _CCCL_THROW_OR_RETURN(__status2, "Failed to get access of a memory pool");
+    return __pool_flags & unsigned{::CU_MEM_ACCESS_FLAGS_PROT_READ};
+  }
+#  endif // _CCCL_CTK_AT_LEAST(12, 2)
+  return false;
+}
+
+/**
+ * @brief Checks if a pointer is a host accessible pointer.
+ *
+ * @param __p The pointer to check.
+ * @return `true` if the pointer is a host accessible pointer, `false` otherwise.
+ */
+[[nodiscard]]
+_CCCL_HOST_API inline bool is_host_accessible(const void* __p)
+{
+  return ::cuda::__is_host_accessible(__p, ::cuda::std::false_type{});
+}
+
+[[nodiscard]]
+_CCCL_HOST_API inline bool __is_host_accessible_nothrow(const void* __p) noexcept
+{
+  return ::cuda::__is_host_accessible(__p, ::cuda::std::true_type{});
+}
+
+/**
+ * @brief Checks if a pointer is a device pointer.
+ *
+ * This internal-only function can be used when the device id is not known.
+ * The main difference between this function and is_device_accessible() is that this function does not check if the
+ * pointer is peer accessible from a specified device.
+ *
+ * @param __p The pointer to check.
+ * @return `true` if the pointer is a device pointer, `false` otherwise.
+ */
+[[nodiscard]]
+_CCCL_HOST_API inline bool __is_device_or_managed_memory(const void* __p) noexcept
+{
+  if (__p == nullptr)
+  {
+    return false;
+  }
+  ::CUpointer_attribute __attrs[4] = {
+    ::CU_POINTER_ATTRIBUTE_MEMORY_TYPE,
+    ::CU_POINTER_ATTRIBUTE_IS_MANAGED,
+    ::CU_POINTER_ATTRIBUTE_DEVICE_ORDINAL,
+    ::CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE};
+  auto __memory_type       = static_cast<::CUmemorytype>(0);
+  int __is_managed         = 0;
+  int __ptr_dev_id         = 0;
+  ::CUmemoryPool __mempool = nullptr;
+  void* __results[4]       = {&__memory_type, &__is_managed, &__ptr_dev_id, &__mempool};
+  const auto __status      = ::cuda::__driver::__pointerGetAttributesNoThrow(__attrs, __results, __p);
+  if (__status != ::cudaSuccess)
+  {
+    return false;
+  }
+  // (1) check if the pointer is unregistered
+  if (__memory_type == static_cast<::CUmemorytype>(0))
+  {
+    return false;
+  }
+  // (2) check if the pointer is managed memory
+  if (__is_managed)
+  {
+    return true;
+  }
+  // (3) check if a memory pool is associated with the pointer
+  if (__mempool != nullptr)
+  {
+    ::CUmemLocation __prop{::CU_MEM_LOCATION_TYPE_DEVICE, __ptr_dev_id};
+    ::CUmemAccess_flags __pool_flags;
+    const auto __status2 = ::cuda::__driver::__mempoolGetAccessNoThrow(__pool_flags, __mempool, &__prop);
+    return (__status2 == ::cudaSuccess) && (static_cast<bool>(__pool_flags));
+  }
+  // (4) check if the pointer is device memory
+  return __memory_type == ::CU_MEMORYTYPE_DEVICE;
+}
+
+template <bool _IsNothrow>
+[[nodiscard]]
+_CCCL_HOST_API inline bool __is_device_accessible(
+  const void* __p, device_ref __device, ::cuda::std::bool_constant<_IsNothrow>) noexcept(_IsNothrow)
+{
+  static_assert(!_IsNothrow, "TODO: implement a no-throw context setter for __is_device_accessible_nothrow");
+  if (__p == nullptr)
+  {
+    return false;
+  }
+
+  const ::cuda::__ensure_current_context __ctx_setter{__device};
+
+  void* __device_ptr = nullptr;
+  const auto __status =
+    ::cuda::__driver::__pointerGetAttributeNoThrow<::CU_POINTER_ATTRIBUTE_DEVICE_POINTER>(__device_ptr, __p);
+  if (__status == ::cudaErrorInvalidValue)
+  {
+    return false;
+  }
+  _CCCL_THROW_OR_RETURN(__status, "Failed to get attributes of a pointer");
+  return __device_ptr != nullptr;
+}
+
+/**
+ * @brief Checks if a pointer is a device accessible pointer.
+ *
+ * @param __p The pointer to check.
+ * @param __device The device to check.
+ * @return `true` if the pointer is a device accessible pointer, `false` otherwise.
+ */
+[[nodiscard]]
+_CCCL_HOST_API inline bool is_device_accessible(const void* __p, device_ref __device)
+{
+  return ::cuda::__is_device_accessible(__p, __device, ::cuda::std::false_type{});
+}
+
+template <class _Dependent = void>
+[[nodiscard]]
+_CCCL_HOST_API inline bool __is_device_accessible_nothrow(const void*, device_ref) noexcept
+{
+  static_assert(::cuda::std::__always_false_v<_Dependent>,
+                "TODO: implement a no-throw context setter for __is_device_accessible_nothrow");
+  return false;
+}
+
+#  undef _CCCL_THROW_OR_RETURN
+
+#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
+
+_CCCL_END_NAMESPACE_CUDA
+
+_CCCL_DIAG_POP
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_IS_POINTER_ACCESSIBLE_H

cccl_upstream/libcudacxx/include/cuda/__memory/is_valid_alignment.h

@@ -0,0 +1,49 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_IS_VALID_ALIGNMENT_H
+#define _CUDA___MEMORY_IS_VALID_ALIGNMENT_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__cmath/pow2.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__type_traits/is_void.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <class _Tp = void>
+[[nodiscard]] _CCCL_API constexpr bool __is_valid_alignment(::cuda::std::size_t __alignment) noexcept
+{
+  if constexpr (::cuda::std::is_void_v<_Tp>)
+  {
+    return __alignment > 0 && ::cuda::is_power_of_two(__alignment);
+  }
+  else
+  {
+    return __alignment >= alignof(_Tp) && ::cuda::is_power_of_two(__alignment);
+  }
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_IS_VALID_ALIGNMENT_H

cccl_upstream/libcudacxx/include/cuda/__memory/ptr_alignment.h

@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_PTR_ALIGNMENT_H
+#define _CUDA___MEMORY_PTR_ALIGNMENT_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__memory/is_valid_alignment.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+//! @brief Get the alignment of a pointer, namely the largest power of two that divides the pointer address.
+//!
+//! @param  __ptr           the input pointer.
+//! @param  __max_alignment the maximum alignment to consider.
+//! @return The alignment of the pointer as a `size_t` value (always a power of two).
+//! @pre    __ptr is not null.
+//! @pre    __max_alignment is a power of two.
+[[nodiscard]] _CCCL_API inline ::cuda::std::size_t
+__ptr_alignment(const void* __ptr, ::cuda::std::size_t __max_alignment) noexcept
+{
+  _CCCL_ASSERT(__ptr != nullptr, "__ptr_alignment requires a non-null pointer");
+  _CCCL_ASSERT(::cuda::__is_valid_alignment(__max_alignment), "invalid __max_alignment value");
+  const auto __addr = reinterpret_cast<::cuda::std::uintptr_t>(__ptr) | __max_alignment;
+  return static_cast<::cuda::std::size_t>(__addr & (~__addr + 1));
+}
+
+[[nodiscard]] _CCCL_API inline ::cuda::std::size_t
+__ptr_alignment(const volatile void* __ptr, ::cuda::std::size_t __max_alignment) noexcept
+{
+  return ::cuda::__ptr_alignment(const_cast<const void*>(__ptr), __max_alignment);
+}
+
+//! @brief Get the alignment of a pointer, namely the largest power of two that divides the pointer address.
+//!
+//! @param  __ptr the input pointer.
+//! @return The alignment of the pointer as a `size_t` value (always a power of two).
+//! @pre    __ptr is not null.
+[[nodiscard]] _CCCL_API inline ::cuda::std::size_t __ptr_alignment(const void* __ptr) noexcept
+{
+  _CCCL_ASSERT(__ptr != nullptr, "__ptr_alignment requires a non-null pointer");
+  const auto __addr = reinterpret_cast<::cuda::std::uintptr_t>(__ptr);
+  return static_cast<::cuda::std::size_t>(__addr & (~__addr + 1));
+}
+
+[[nodiscard]] _CCCL_API inline ::cuda::std::size_t __ptr_alignment(const volatile void* __ptr) noexcept
+{
+  return ::cuda::__ptr_alignment(const_cast<const void*>(__ptr));
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_PTR_ALIGNMENT_H

cccl_upstream/libcudacxx/include/cuda/__memory/ptr_in_range.h

@@ -0,0 +1,92 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_POINTER_IN_RANGE_H
+#define _CUDA___MEMORY_POINTER_IN_RANGE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/cstdint>
+#if _CCCL_HOST_COMPILATION()
+#  include <functional>
+#endif // _CCCL_HOST_COMPILATION()
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+// Pointers comparison <, <=, >=, > is undefined behavior in C++ (https://eel.is/c++draft/expr.rel#4) when pointers
+// don't belong to the same object or array.
+// - Even when a platform guarantees flat address space, the compiler can leverage UB for optimization purposes.
+// - However, the compiler treats ::std::less<> other functional operators in a special way, ensuring a total ordering.
+// - For device code, we can convert pointers to uintptr_t and compare them.
+//
+// References:
+// - https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2024/p3234r0.html
+// - https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/p2865r2.pdf
+// - https://www.boost.org/doc/libs/develop/libs/core/doc/html/core/pointer_in_range.html
+// - https://pvs-studio.com/en/blog/posts/cpp/1199/
+// - https://releases.llvm.org/20.1.0/tools/clang/docs/ReleaseNotes.html#resolutions-to-c-defect-reports
+
+#if _CCCL_HOST_COMPILATION()
+
+template <typename _Tp>
+[[nodiscard]] _CCCL_API bool __ptr_in_range_host(_Tp* __ptr, _Tp* __start, _Tp* __end) noexcept
+{
+  _CCCL_ASSERT(::std::greater_equal<>{}(__end, __start), "__ptr_in_range_host: __end must be greater than __start");
+  return ::std::greater_equal<>{}(__ptr, __start) && ::std::less<>{}(__ptr, __end);
+}
+
+#endif // _CCCL_HOST_COMPILATION()
+
+#if _CCCL_DEVICE_COMPILATION()
+
+template <typename _Tp>
+[[nodiscard]] _CCCL_API bool __ptr_in_range_device(_Tp* __ptr, _Tp* __start, _Tp* __end) noexcept
+{
+  using uintptr_t  = ::cuda::std::uintptr_t;
+  auto __end_ptr   = reinterpret_cast<uintptr_t>(__end);
+  auto __start_ptr = reinterpret_cast<uintptr_t>(__start);
+  auto __ptr_ptr   = reinterpret_cast<uintptr_t>(__ptr);
+  _CCCL_ASSERT(__end_ptr >= __start_ptr, "__ptr_in_range_device: __end must be greater than __start");
+  return __ptr_ptr >= __start_ptr && __ptr_ptr < __end_ptr;
+}
+
+#endif // _CCCL_DEVICE_COMPILATION()
+
+template <typename _Tp>
+[[nodiscard]] _CCCL_API constexpr bool ptr_in_range(_Tp* __ptr, _Tp* __start, _Tp* __end) noexcept
+{
+  _CCCL_IF_CONSTEVAL_DEFAULT
+  {
+    _CCCL_ASSERT(__end >= __start, "ptr_in_range: __end must be greater than __start");
+    return __ptr >= __start && __ptr < __end; // UB is not possible in a constant expression
+  }
+  else
+  {
+    NV_IF_ELSE_TARGET(NV_IS_HOST,
+                      (return ::cuda::__ptr_in_range_host(__ptr, __start, __end);),
+                      (return ::cuda::__ptr_in_range_device(__ptr, __start, __end);));
+  }
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_POINTER_IN_RANGE_H

cccl_upstream/libcudacxx/include/cuda/__memory/ptr_rebind.h

@@ -0,0 +1,75 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_PTR_REBIND_H
+#define _CUDA___MEMORY_PTR_REBIND_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__memory/assume_aligned.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__type_traits/is_void.h>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _Up, typename _Tp>
+[[nodiscard]] _CCCL_HOST_DEVICE_API _Up* ptr_rebind(_Tp* __ptr) noexcept
+{
+  if constexpr (::cuda::std::is_same_v<_Up, _Tp>) // also handle _Tp == _Up == void
+  {
+    return __ptr;
+  }
+  else if constexpr (::cuda::std::is_void_v<_Up>) // _Tp: non-void, _Up: void
+  {
+    _CCCL_ASSERT(reinterpret_cast<::cuda::std::uintptr_t>(__ptr) % alignof(_Tp) == 0, "ptr is not aligned");
+    return ::cuda::std::assume_aligned<alignof(_Tp)>(reinterpret_cast<_Up*>(__ptr));
+  }
+  else
+  {
+    constexpr auto __max_alignment = alignof(_Up) > alignof(_Tp) ? alignof(_Up) : alignof(_Tp);
+    _CCCL_ASSERT(reinterpret_cast<::cuda::std::uintptr_t>(__ptr) % __max_alignment == 0, "ptr is not aligned");
+    return ::cuda::std::assume_aligned<__max_alignment>(reinterpret_cast<_Up*>(__ptr));
+  }
+}
+
+template <typename _Up, typename _Tp>
+[[nodiscard]] _CCCL_HOST_DEVICE_API const _Up* ptr_rebind(const _Tp* __ptr) noexcept
+{
+  return ::cuda::ptr_rebind<const _Up>(const_cast<_Tp*>(__ptr));
+}
+
+template <typename _Up, typename _Tp>
+[[nodiscard]] _CCCL_HOST_DEVICE_API volatile _Up* ptr_rebind(volatile _Tp* __ptr) noexcept
+{
+  return ::cuda::ptr_rebind<volatile _Up>(const_cast<_Tp*>(__ptr));
+}
+
+template <typename _Up, typename _Tp>
+[[nodiscard]] _CCCL_HOST_DEVICE_API const volatile _Up* ptr_rebind(const volatile _Tp* __ptr) noexcept
+{
+  return ::cuda::ptr_rebind<const volatile _Up>(const_cast<_Tp*>(__ptr));
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_PTR_REBIND_H

cccl_upstream/libcudacxx/include/cuda/__memory/ranges_overlap.h

@@ -0,0 +1,121 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_RANGES_OVERLAP_H
+#define _CUDA___MEMORY_RANGES_OVERLAP_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__iterator/concepts.h>
+#include <cuda/std/__memory/pointer_traits.h>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__cccl/prologue.h>
+
+#if _CCCL_HOST_COMPILATION()
+#  include <functional>
+#endif // _CCCL_HOST_COMPILATION()
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+#if _CCCL_CUDA_COMPILATION()
+[[nodiscard]] _CCCL_DEVICE_API inline bool __ptr_ranges_overlap_device(
+  const void* __lhs_begin, const void* __lhs_end, const void* __rhs_begin, const void* __rhs_end) noexcept
+{
+  using uintptr_t            = ::cuda::std::uintptr_t;
+  const auto __lhs_start_ptr = reinterpret_cast<uintptr_t>(__lhs_begin);
+  const auto __lhs_end_ptr   = reinterpret_cast<uintptr_t>(__lhs_end);
+  const auto __rhs_start_ptr = reinterpret_cast<uintptr_t>(__rhs_begin);
+  const auto __rhs_end_ptr   = reinterpret_cast<uintptr_t>(__rhs_end);
+  _CCCL_ASSERT(__lhs_start_ptr <= __lhs_end_ptr, "lhs range is invalid");
+  _CCCL_ASSERT(__rhs_start_ptr <= __rhs_end_ptr, "rhs range is invalid");
+  return __lhs_start_ptr < __rhs_end_ptr && __rhs_start_ptr < __lhs_end_ptr;
+}
+#endif // _CCCL_CUDA_COMPILATION()
+
+#if !_CCCL_COMPILER(NVRTC)
+template <typename _Tp>
+[[nodiscard]] _CCCL_HOST_API bool
+__ptr_ranges_overlap_host(_Tp* __lhs_begin, _Tp* __lhs_end, _Tp* __rhs_begin, _Tp* __rhs_end) noexcept
+{
+  _CCCL_ASSERT(::std::less_equal<>{}(__lhs_begin, __lhs_end), "lhs range is invalid");
+  _CCCL_ASSERT(::std::less_equal<>{}(__rhs_begin, __rhs_end), "rhs range is invalid");
+  return ::std::less<>{}(__lhs_begin, __rhs_end) && ::std::less<>{}(__rhs_begin, __lhs_end);
+}
+#endif // !_CCCL_COMPILER(NVRTC)
+
+_CCCL_TEMPLATE(typename _Tp)
+_CCCL_REQUIRES(::cuda::std::forward_iterator<_Tp>)
+[[nodiscard]] _CCCL_API constexpr bool
+ranges_overlap(_Tp __lhs_begin, _Tp __lhs_end, _Tp __rhs_begin, _Tp __rhs_end) noexcept
+{
+  if constexpr (::cuda::std::contiguous_iterator<_Tp>)
+  {
+    _CCCL_IF_CONSTEVAL_DEFAULT
+    {
+      // UB is not possible in a constant expression
+      _CCCL_ASSERT(__lhs_begin <= __lhs_end, "lhs range is invalid");
+      _CCCL_ASSERT(__rhs_begin <= __rhs_end, "rhs range is invalid");
+      return __lhs_begin < __rhs_end && __rhs_begin < __lhs_end;
+    }
+    else
+    {
+      const auto __ptr_lhs_begin = ::cuda::std::to_address(__lhs_begin);
+      const auto __ptr_lhs_end   = ::cuda::std::to_address(__lhs_end);
+      const auto __ptr_rhs_begin = ::cuda::std::to_address(__rhs_begin);
+      const auto __ptr_rhs_end   = ::cuda::std::to_address(__rhs_end);
+      NV_IF_ELSE_TARGET(
+        NV_IS_HOST,
+        (return ::cuda::__ptr_ranges_overlap_host(__ptr_lhs_begin, __ptr_lhs_end, __ptr_rhs_begin, __ptr_rhs_end);),
+        (return ::cuda::__ptr_ranges_overlap_device(__ptr_lhs_begin, __ptr_lhs_end, __ptr_rhs_begin, __ptr_rhs_end);));
+    }
+  }
+  else if constexpr (::cuda::std::random_access_iterator<_Tp>)
+  {
+    _CCCL_ASSERT(__lhs_begin <= __lhs_end, "lhs range is invalid");
+    _CCCL_ASSERT(__rhs_begin <= __rhs_end, "rhs range is invalid");
+    return __lhs_begin < __rhs_end && __rhs_begin < __lhs_end;
+  }
+  else
+  {
+    // For forward iterators: if two ranges [A,B) and [C,D) overlap from the same sequence,
+    // then either C is in [A,B) or A is in [C,D). We check both conditions.
+    for (auto __lhs_it = __lhs_begin; __lhs_it != __lhs_end; ++__lhs_it)
+    {
+      if (__lhs_it == __rhs_begin)
+      {
+        return true;
+      }
+    }
+    for (auto __rhs_it = __rhs_begin; __rhs_it != __rhs_end; ++__rhs_it)
+    {
+      if (__rhs_it == __lhs_begin)
+      {
+        return true;
+      }
+    }
+    return false;
+  }
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_RANGES_OVERLAP_H

cccl_upstream/libcudacxx/include/cuda/__memory/uninitialized_array.h

@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MEMORY_UNINITIALIZED_ARRAY_H
+#define _CUDA___MEMORY_UNINITIALIZED_ARRAY_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__new/launder.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+//! Aligned storage for _Tp elements (not constructed).
+//! Initialize before use with the `data()` method
+template <class _Tp, size_t _Size, size_t _Alignment = alignof(_Tp)>
+struct __uninitialized_array
+{
+  alignas(_Alignment) unsigned char __data[_Size * sizeof(_Tp)];
+
+  [[nodiscard]] _CCCL_API _Tp* data() noexcept
+  {
+    return ::cuda::std::launder(reinterpret_cast<_Tp*>(__data));
+  }
+
+  [[nodiscard]] _CCCL_API const _Tp* data() const noexcept
+  {
+    return ::cuda::std::launder(reinterpret_cast<const _Tp*>(__data));
+  }
+
+  [[nodiscard]] _CCCL_API _Tp& operator[](const size_t __idx) noexcept
+  {
+    _CCCL_ASSERT(__idx < _Size, "out of bounds access in uninitialized_array::operator[]");
+    return data()[__idx];
+  }
+
+  [[nodiscard]] _CCCL_API const _Tp& operator[](const size_t __idx) const noexcept
+  {
+    _CCCL_ASSERT(__idx < _Size, "out of bounds access in uninitialized_array::operator[]");
+    return data()[__idx];
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MEMORY_UNINITIALIZED_ARRAY_H