[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/libcudacxx/include/cuda/__mdspan/dlpack_to_mdspan.h

@@ -0,0 +1,309 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the libcu++ Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MDSPAN_DLPACK_TO_MDSPAN_H
+#define _CUDA___MDSPAN_DLPACK_TO_MDSPAN_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#if _CCCL_HAS_DLPACK()
+
+#  include <cuda/__internal/dlpack.h>
+
+#  if _CCCL_HAS_DLPACK_VERSION_1()
+#    include <cuda/__mdspan/host_device_mdspan.h>
+#    include <cuda/__mdspan/layout_stride_relaxed.h>
+#    include <cuda/__mdspan/mdspan_to_dlpack.h>
+#    include <cuda/__mdspan/traits.h>
+#    include <cuda/__memory/is_aligned.h>
+#    include <cuda/__numeric/mul_overflow.h>
+#    include <cuda/mdspan>
+#    include <cuda/std/__cstddef/types.h>
+#    include <cuda/std/__exception/exception_macros.h>
+#    include <cuda/std/__host_stdlib/stdexcept>
+#    include <cuda/std/__utility/cmp.h>
+#    include <cuda/std/array>
+#    include <cuda/std/cstdint>
+
+#    include <dlpack/dlpack.h>
+//
+#    include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <::cuda::std::size_t _Rank>
+using __dlpack_extents_type = ::cuda::std::dims<_Rank, ::cuda::std::int64_t>;
+
+template <typename _ElementType>
+[[nodiscard]] _CCCL_HOST_API inline bool __validate_dlpack_data_type(const ::DLDataType& __dtype) noexcept
+{
+  const auto __expected = ::cuda::__data_type_to_dlpack<_ElementType>();
+  return __dtype.code == __expected.code && __dtype.bits == __expected.bits && __dtype.lanes == __expected.lanes;
+}
+
+[[nodiscard]]
+_CCCL_HOST_API inline ::cuda::std::int64_t
+__get_layout_right_stride(const ::cuda::std::int64_t* __shapes, ::cuda::std::size_t __pos, ::cuda::std::size_t __rank)
+{
+  ::cuda::std::int64_t __stride = 1;
+  for (auto __i = __pos + 1; __i < __rank; ++__i)
+  {
+    if (::cuda::mul_overflow(__stride, __stride, __shapes[__i]))
+    {
+      _CCCL_THROW(::std::invalid_argument, "shape overflow");
+    }
+  }
+  return __stride;
+}
+
+[[nodiscard]]
+_CCCL_HOST_API inline ::cuda::std::int64_t
+__get_layout_left_stride(const ::cuda::std::int64_t* __shapes, ::cuda::std::size_t __pos)
+{
+  ::cuda::std::int64_t __stride = 1;
+  for (::cuda::std::size_t __i = 0; __i < __pos; ++__i)
+  {
+    if (::cuda::mul_overflow(__stride, __stride, __shapes[__i]))
+    {
+      _CCCL_THROW(::std::invalid_argument, "shape overflow");
+    }
+  }
+  return __stride;
+}
+
+template <typename _LayoutPolicy>
+_CCCL_HOST_API void __validate_dlpack_strides(const ::DLTensor& __tensor, [[maybe_unused]] ::cuda::std::size_t __rank)
+{
+  const auto __strides_ptr = __tensor.strides;
+  if (__strides_ptr == nullptr)
+  {
+#    if _CCCL_DLPACK_AT_LEAST(1, 2)
+    _CCCL_THROW(::std::invalid_argument, "strides=nullptr is not supported for DLPack v1.2 and later");
+#    else
+    // strides == nullptr means row-major (C-contiguous) layout
+    if (__is_layout_left && __rank > 1)
+    {
+      _CCCL_THROW(::std::invalid_argument, "strides must be non-null for layout_left");
+    }
+    else
+    {
+      return;
+    }
+#    endif // _CCCL_DLPACK_AT_LEAST(1, 2)
+  }
+  for (::cuda::std::size_t __pos = 0; __pos < __rank; ++__pos)
+  {
+    if constexpr (::cuda::__is_layout_right_v<_LayoutPolicy>)
+    {
+      if (__strides_ptr[__pos] != ::cuda::__get_layout_right_stride(__tensor.shape, __pos, __rank))
+      {
+        _CCCL_THROW(::std::invalid_argument, "DLTensor strides are not compatible with layout_right");
+      }
+    }
+    else if constexpr (::cuda::__is_layout_left_v<_LayoutPolicy>)
+    {
+      if (__strides_ptr[__pos] != ::cuda::__get_layout_left_stride(__tensor.shape, __pos))
+      {
+        _CCCL_THROW(::std::invalid_argument, "DLTensor strides are not compatible with layout_left");
+      }
+    }
+    else if constexpr (::cuda::__is_layout_stride_v<_LayoutPolicy>)
+    {
+      if (__strides_ptr[__pos] <= 0)
+      {
+        _CCCL_THROW(::std::invalid_argument, "layout_stride requires strictly positive strides");
+      }
+    }
+    // layout_stride_relaxed accepts any valid stride
+  }
+}
+
+template <typename _LayoutPolicy, ::cuda::std::size_t _Rank>
+[[nodiscard]]
+_CCCL_HOST_API constexpr auto __get_layout_mapping_type() noexcept
+{
+  if constexpr (::cuda::__is_layout_stride_relaxed_v<_LayoutPolicy>)
+  {
+    using __strides_type = ::cuda::steps<_Rank, ::cuda::std::int64_t>;
+    return ::cuda::layout_stride_relaxed::mapping<__dlpack_extents_type<_Rank>, __strides_type>{};
+  }
+  else
+  {
+    return typename _LayoutPolicy::template mapping<__dlpack_extents_type<_Rank>>{};
+  }
+}
+
+template <typename _ElementType, ::cuda::std::size_t _Rank, typename _LayoutPolicy>
+[[nodiscard]]
+_CCCL_HOST_API ::cuda::std::mdspan<_ElementType, __dlpack_extents_type<_Rank>, _LayoutPolicy>
+__to_mdspan(const ::DLTensor& __tensor)
+{
+  using ::cuda::std::int64_t;
+  using __mdspan_type  = ::cuda::std::mdspan<_ElementType, __dlpack_extents_type<_Rank>, _LayoutPolicy>;
+  using __mapping_type = decltype(::cuda::__get_layout_mapping_type<_LayoutPolicy, _Rank>());
+  using __element_type = typename __mdspan_type::element_type;
+  // TODO(fbusato): add support for layout_right_padded, layout_left_padded
+  static_assert(::cuda::__is_cuda_mdspan_layout_v<_LayoutPolicy>, "Unsupported layout policy");
+  if (::cuda::std::cmp_not_equal(__tensor.ndim, _Rank))
+  {
+    _CCCL_THROW(::std::invalid_argument, "DLTensor rank does not match expected rank");
+  }
+  if (!::cuda::__validate_dlpack_data_type<__element_type>(__tensor.dtype))
+  {
+    _CCCL_THROW(::std::invalid_argument, "DLTensor data type does not match expected type");
+  }
+  if (__tensor.data == nullptr)
+  {
+    _CCCL_THROW(::std::invalid_argument, "DLTensor data must be non-null");
+  }
+  // (1) Evaluate Data Pointer
+  const auto __datatype_size = __tensor.dtype.bits * __tensor.dtype.lanes / 8;
+  __element_type* __data     = nullptr;
+  if constexpr (::cuda::__is_layout_stride_relaxed_v<_LayoutPolicy>)
+  {
+    if (__datatype_size > 0 && __tensor.byte_offset % __datatype_size != 0)
+    {
+      _CCCL_THROW(::std::invalid_argument, "DLTensor byte_offset must be a multiple of element size");
+    }
+    __data = reinterpret_cast<__element_type*>(__tensor.data);
+  }
+  else
+  {
+    __data = reinterpret_cast<__element_type*>(static_cast<char*>(__tensor.data) + __tensor.byte_offset);
+  }
+  // this is not the exact solution because data type size != data type alignment.
+  // However, it always works for the supported data types.
+  if (__datatype_size > 0 && !::cuda::is_aligned(__data, __datatype_size))
+  {
+    _CCCL_THROW(::std::invalid_argument, "DLTensor data must be aligned to the data type");
+  }
+  // Rank 0 case
+  if constexpr (_Rank == 0)
+  {
+    if constexpr (::cuda::__is_layout_stride_relaxed_v<_LayoutPolicy>)
+    {
+      const auto __element_offset = static_cast<int64_t>(__tensor.byte_offset / sizeof(__element_type));
+      return __mdspan_type{__data, __mapping_type{{}, {}, __element_offset}};
+    }
+    else
+    {
+      return __mdspan_type{__data, __mapping_type{}};
+    }
+  }
+  else // Rank > 0
+  {
+    // (2) Evaluate Extents
+    if (__tensor.shape == nullptr)
+    {
+      _CCCL_THROW(::std::invalid_argument, "DLTensor shape must be non-null");
+    }
+    ::cuda::std::array<int64_t, _Rank> __extents_array{};
+    for (::cuda::std::size_t __i = 0; __i < _Rank; ++__i)
+    {
+      if (__tensor.shape[__i] < 0)
+      {
+        _CCCL_THROW(::std::invalid_argument, "DLTensor shapes must be positive");
+      }
+      __extents_array[__i] = __tensor.shape[__i];
+    }
+    // (3) Evaluate Strides
+    ::cuda::__validate_dlpack_strides<_LayoutPolicy>(__tensor, _Rank);
+    if constexpr (::cuda::__is_layout_stride_v<_LayoutPolicy> || ::cuda::__is_layout_stride_relaxed_v<_LayoutPolicy>)
+    {
+      ::cuda::std::array<int64_t, _Rank> __strides_array{};
+      for (::cuda::std::size_t __i = 0; __i < _Rank; ++__i)
+      {
+        const bool __has_strides = __tensor.strides != nullptr;
+        __strides_array[__i] =
+          __has_strides ? __tensor.strides[__i] : ::cuda::__get_layout_right_stride(__tensor.shape, __i, _Rank);
+      }
+      if constexpr (::cuda::__is_layout_stride_relaxed_v<_LayoutPolicy>)
+      {
+        const auto __element_offset = static_cast<int64_t>(__tensor.byte_offset / sizeof(__element_type));
+        return __mdspan_type{__data, __mapping_type{__extents_array, __strides_array, __element_offset}};
+      }
+      else
+      {
+        return __mdspan_type{__data, __mapping_type{__extents_array, __strides_array}};
+      }
+    }
+    else
+    {
+      return __mdspan_type{__data, __dlpack_extents_type<_Rank>{__extents_array}};
+    }
+  }
+}
+
+/***********************************************************************************************************************
+ * Public API
+ **********************************************************************************************************************/
+
+//! @brief Converts a DLTensor on host memory to a \c host_mdspan
+//! @param __tensor The DLTensor to convert. Must have device type \c kDLCPU
+//! @return A \c host_mdspan viewing the tensor data with the specified element type, rank, and layout
+template <typename _ElementType, ::cuda::std::size_t _Rank, typename _LayoutPolicy = ::cuda::layout_stride_relaxed>
+[[nodiscard]]
+_CCCL_HOST_API ::cuda::host_mdspan<_ElementType, __dlpack_extents_type<_Rank>, _LayoutPolicy>
+to_host_mdspan(const ::DLTensor& __tensor)
+{
+  if (__tensor.device.device_type != ::kDLCPU)
+  {
+    _CCCL_THROW(::std::invalid_argument, "DLTensor device type must be kDLCPU for host_mdspan");
+  }
+  using __mdspan_type = ::cuda::host_mdspan<_ElementType, __dlpack_extents_type<_Rank>, _LayoutPolicy>;
+  return __mdspan_type{::cuda::__to_mdspan<_ElementType, _Rank, _LayoutPolicy>(__tensor)};
+}
+
+//! @brief Converts a DLTensor on device memory to a \c device_mdspan
+//! @param __tensor The DLTensor to convert. Must have device type \c kDLCUDA
+//! @return A \c device_mdspan viewing the tensor data with the specified element type, rank, and layout
+template <typename _ElementType, ::cuda::std::size_t _Rank, typename _LayoutPolicy = ::cuda::layout_stride_relaxed>
+[[nodiscard]]
+_CCCL_HOST_API ::cuda::device_mdspan<_ElementType, __dlpack_extents_type<_Rank>, _LayoutPolicy>
+to_device_mdspan(const ::DLTensor& __tensor)
+{
+  if (__tensor.device.device_type != ::kDLCUDA)
+  {
+    _CCCL_THROW(::std::invalid_argument, "DLTensor device type must be kDLCUDA for device_mdspan");
+  }
+  using __mdspan_type = ::cuda::device_mdspan<_ElementType, __dlpack_extents_type<_Rank>, _LayoutPolicy>;
+  return __mdspan_type{::cuda::__to_mdspan<_ElementType, _Rank, _LayoutPolicy>(__tensor)};
+}
+
+//! @brief Converts a DLTensor on managed memory to a \c managed_mdspan
+//! @param __tensor The DLTensor to convert. Must have device type \c kDLCUDAManaged
+//! @return A \c managed_mdspan viewing the tensor data with the specified element type, rank, and layout
+template <typename _ElementType, ::cuda::std::size_t _Rank, typename _LayoutPolicy = ::cuda::layout_stride_relaxed>
+[[nodiscard]]
+_CCCL_HOST_API ::cuda::managed_mdspan<_ElementType, __dlpack_extents_type<_Rank>, _LayoutPolicy>
+to_managed_mdspan(const ::DLTensor& __tensor)
+{
+  if (__tensor.device.device_type != ::kDLCUDAManaged)
+  {
+    _CCCL_THROW(::std::invalid_argument, "DLTensor device type must be kDLCUDAManaged for managed_mdspan");
+  }
+  using __mdspan_type = ::cuda::managed_mdspan<_ElementType, __dlpack_extents_type<_Rank>, _LayoutPolicy>;
+  return __mdspan_type{::cuda::__to_mdspan<_ElementType, _Rank, _LayoutPolicy>(__tensor)};
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#    include <cuda/std/__cccl/epilogue.h>
+
+#  endif // _CCCL_HAS_DLPACK_VERSION_1()
+#endif // __CCCL_HAS_DLPACK()
+#endif // _CUDA___MDSPAN_DLPACK_TO_MDSPAN_H

cccl_upstream/libcudacxx/include/cuda/__mdspan/host_device_accessor.h

@@ -0,0 +1,515 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MDSPAN_HOST_DEVICE_ACCESSOR_H
+#define _CUDA___MDSPAN_HOST_DEVICE_ACCESSOR_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__driver/driver_api.h>
+#include <cuda/__memory/address_space.h>
+#include <cuda/__memory/is_pointer_accessible.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__iterator/concepts.h>
+#include <cuda/std/__memory/pointer_traits.h>
+#include <cuda/std/__type_traits/is_constructible.h>
+#include <cuda/std/__type_traits/is_convertible.h>
+#include <cuda/std/__type_traits/is_default_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
+#include <cuda/std/__utility/declval.h>
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _Accessor>
+class __host_accessor;
+
+template <typename _Accessor>
+class __device_accessor;
+
+template <typename _Accessor>
+class __managed_accessor;
+
+template <typename _Accessor>
+using host_accessor = __host_accessor<_Accessor>;
+
+template <typename _Accessor>
+using device_accessor = __device_accessor<_Accessor>;
+
+template <typename _Accessor>
+using managed_accessor = __managed_accessor<_Accessor>;
+
+/***********************************************************************************************************************
+ * Host/Device/Managed Accessor Traits
+ **********************************************************************************************************************/
+
+template <typename>
+inline constexpr bool is_host_accessor_v = false;
+
+template <typename>
+inline constexpr bool is_device_accessor_v = false;
+
+template <typename>
+inline constexpr bool is_managed_accessor_v = false;
+
+template <typename _Accessor>
+inline constexpr bool is_host_accessor_v<__host_accessor<_Accessor>> = true;
+
+template <typename _Accessor>
+inline constexpr bool is_device_accessor_v<__device_accessor<_Accessor>> = true;
+
+template <typename _Accessor>
+inline constexpr bool is_managed_accessor_v<__managed_accessor<_Accessor>> = true;
+
+template <typename _Tp>
+inline constexpr bool is_host_device_managed_accessor_v =
+  is_host_accessor_v<_Tp> || is_device_accessor_v<_Tp> || is_managed_accessor_v<_Tp>;
+
+/***********************************************************************************************************************
+ * Host Accessor
+ **********************************************************************************************************************/
+
+template <typename _Accessor>
+class __host_accessor : public _Accessor
+{
+  static_assert(!is_host_device_managed_accessor_v<_Accessor>,
+                "cuda::__host_accessor/cuda::__device_accessor/cuda::__managed_accessor cannot be nested");
+
+  using __data_handle_type = typename _Accessor::data_handle_type;
+
+  static constexpr bool __is_access_noexcept =
+    noexcept(::cuda::std::declval<_Accessor>().access(::cuda::std::declval<__data_handle_type>(), 0));
+
+  static constexpr bool __is_offset_noexcept =
+    noexcept(::cuda::std::declval<_Accessor>().offset(::cuda::std::declval<__data_handle_type>(), 0));
+
+#if !_CCCL_COMPILER(NVRTC)
+  [[nodiscard]]
+  _CCCL_HOST_API static bool __is_host_accessible_pointer([[maybe_unused]] __data_handle_type __p) noexcept
+  {
+#  if _CCCL_HAS_CTK()
+    if constexpr (::cuda::std::contiguous_iterator<__data_handle_type>)
+    {
+      return ::cuda::__is_host_accessible_nothrow(::cuda::std::to_address(__p));
+    }
+    else
+#  endif // _CCCL_HAS_CTK()
+    {
+      return true; // cannot be verified
+    }
+  }
+#endif // !_CCCL_COMPILER(NVRTC)
+
+public:
+  using offset_policy    = __host_accessor<typename _Accessor::offset_policy>;
+  using data_handle_type = __data_handle_type;
+  using reference        = typename _Accessor::reference;
+  using element_type     = typename _Accessor::element_type;
+
+  _CCCL_TEMPLATE(class _Accessor2 = _Accessor)
+  _CCCL_REQUIRES(::cuda::std::is_default_constructible_v<_Accessor2>)
+  _CCCL_API constexpr __host_accessor() noexcept(::cuda::std::is_nothrow_default_constructible_v<_Accessor2>)
+      : _Accessor{}
+  {}
+
+  _CCCL_API constexpr __host_accessor(_Accessor&& __acc) noexcept(
+    ::cuda::std::is_nothrow_move_constructible_v<_Accessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {}
+
+  _CCCL_API constexpr __host_accessor(const _Accessor& __acc) noexcept(
+    ::cuda::std::is_nothrow_copy_constructible_v<_Accessor>)
+      : _Accessor{__acc}
+  {}
+
+  template <typename _OtherAccessor>
+  __host_accessor(const __device_accessor<_OtherAccessor>&) = delete;
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    ::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr __host_accessor(const __host_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    !::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr explicit __host_accessor(const __host_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, _OtherAccessor> _CCCL_AND(
+    ::cuda::std::is_convertible_v<_OtherAccessor, _Accessor>))
+  _CCCL_API constexpr __host_accessor(__host_accessor<_OtherAccessor>&& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, _OtherAccessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    !::cuda::std::is_convertible_v<_OtherAccessor, _Accessor>))
+  _CCCL_API constexpr explicit __host_accessor(__host_accessor<_OtherAccessor>&& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, _OtherAccessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    ::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr __host_accessor(const __managed_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    !::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr explicit __host_accessor(const __managed_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {}
+
+  _CCCL_API constexpr reference access(data_handle_type __p, ::cuda::std::size_t __i) const
+    noexcept(__is_access_noexcept)
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      NV_IF_TARGET(NV_IS_DEVICE, (_CCCL_VERIFY(false, "cuda::__host_accessor cannot be used in DEVICE code");))
+    }
+    return _Accessor::access(__p, __i);
+  }
+
+  [[nodiscard]] _CCCL_API constexpr data_handle_type offset(data_handle_type __p, ::cuda::std::size_t __i) const
+    noexcept(__is_offset_noexcept)
+  {
+    return _Accessor::offset(__p, __i);
+  }
+
+#if !defined(_CCCL_DISABLE_MDSPAN_ACCESSOR_DETECT_INVALIDITY)
+  [[nodiscard]] _CCCL_API constexpr bool
+  __detectably_invalid([[maybe_unused]] data_handle_type __p, ::cuda::std::size_t) const noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      bool __is_valid = true;
+      NV_IF_TARGET(NV_IS_HOST, (__is_valid = __is_host_accessible_pointer(__p);), (__is_valid = false;))
+      _CCCL_ASSERT(__is_valid, "host_accessor (mdspan): data handle doesn't point to a valid host memory");
+      return !__is_valid;
+    }
+    return false;
+  }
+#endif // !defined(_CCCL_DISABLE_MDSPAN_ACCESSOR_DETECT_INVALIDITY)
+};
+
+/***********************************************************************************************************************
+ * Device Accessor
+ **********************************************************************************************************************/
+
+template <typename _Accessor>
+class __device_accessor : public _Accessor
+{
+  static_assert(!is_host_device_managed_accessor_v<_Accessor>,
+                "cuda::__host_accessor/cuda::__device_accessor/cuda::__managed_accessor cannot be nested");
+
+  using __data_handle_type = typename _Accessor::data_handle_type;
+
+  static constexpr bool __is_access_noexcept =
+    noexcept(::cuda::std::declval<_Accessor>().access(::cuda::std::declval<__data_handle_type>(), 0));
+
+  static constexpr bool __is_offset_noexcept =
+    noexcept(::cuda::std::declval<_Accessor>().offset(::cuda::std::declval<__data_handle_type>(), 0));
+
+  [[nodiscard]] _CCCL_API static bool
+  __is_device_accessible_pointer_from_host([[maybe_unused]] __data_handle_type __p) noexcept
+  {
+#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
+    if constexpr (::cuda::std::contiguous_iterator<__data_handle_type>)
+    {
+      return ::cuda::__is_device_or_managed_memory(::cuda::std::to_address(__p));
+    }
+    else
+#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
+    {
+      return true; // cannot be verified
+    }
+  }
+
+public:
+  using offset_policy    = __device_accessor<typename _Accessor::offset_policy>;
+  using data_handle_type = __data_handle_type;
+  using reference        = typename _Accessor::reference;
+  using element_type     = typename _Accessor::element_type;
+
+  _CCCL_TEMPLATE(class _Accessor2 = _Accessor)
+  _CCCL_REQUIRES(::cuda::std::is_default_constructible_v<_Accessor2>)
+  _CCCL_API constexpr __device_accessor() noexcept(::cuda::std::is_nothrow_default_constructible_v<_Accessor2>)
+      : _Accessor{}
+  {}
+
+  _CCCL_API constexpr __device_accessor(_Accessor&& __acc) noexcept(
+    ::cuda::std::is_nothrow_move_constructible_v<_Accessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {}
+
+  _CCCL_API constexpr __device_accessor(const _Accessor& __acc) noexcept(
+    ::cuda::std::is_nothrow_copy_constructible_v<_Accessor>)
+      : _Accessor{__acc}
+  {}
+
+  template <typename _OtherAccessor>
+  __device_accessor(const __host_accessor<_OtherAccessor>&) = delete;
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    ::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr __device_accessor(const __device_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    !::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr explicit __device_accessor(const __device_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, _OtherAccessor> _CCCL_AND(
+    ::cuda::std::is_convertible_v<_OtherAccessor, _Accessor>))
+  _CCCL_API constexpr __device_accessor(__device_accessor<_OtherAccessor>&& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, _OtherAccessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, _OtherAccessor> _CCCL_AND(
+    !::cuda::std::is_convertible_v<_OtherAccessor, _Accessor>))
+  _CCCL_API constexpr explicit __device_accessor(__device_accessor<_OtherAccessor>&& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, _OtherAccessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    ::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr __device_accessor(const __managed_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    !::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr explicit __device_accessor(const __managed_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {}
+
+  _CCCL_API constexpr reference access(data_handle_type __p, ::cuda::std::size_t __i) const
+    noexcept(__is_access_noexcept)
+  {
+    return _Accessor::access(__p, __i);
+  }
+
+  [[nodiscard]] _CCCL_API constexpr data_handle_type offset(data_handle_type __p, ::cuda::std::size_t __i) const
+    noexcept(__is_offset_noexcept)
+  {
+    return _Accessor::offset(__p, __i);
+  }
+
+#if !defined(_CCCL_DISABLE_MDSPAN_ACCESSOR_DETECT_INVALIDITY)
+  [[nodiscard]] _CCCL_API constexpr bool
+  __detectably_invalid([[maybe_unused]] data_handle_type __p, ::cuda::std::size_t) const noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      bool __is_valid = true;
+      NV_IF_TARGET(NV_IS_HOST, (__is_valid = __is_device_accessible_pointer_from_host(__p);))
+      _CCCL_ASSERT(__is_valid,
+                   "device_accessor (mdspan): data handle doesn't point to a valid device or managed memory");
+      return !__is_valid;
+    }
+    return false;
+  }
+#endif // !defined(_CCCL_DISABLE_MDSPAN_ACCESSOR_DETECT_INVALIDITY)
+};
+
+/***********************************************************************************************************************
+ * Managed Accessor
+ **********************************************************************************************************************/
+
+template <typename _Accessor>
+class __managed_accessor : public _Accessor
+{
+  static_assert(!is_host_device_managed_accessor_v<_Accessor>,
+                "cuda::__host_accessor/cuda::__device_accessor/cuda::__managed_accessor cannot be nested");
+
+  using __data_handle_type = typename _Accessor::data_handle_type;
+
+  static constexpr bool __is_access_noexcept =
+    noexcept(::cuda::std::declval<_Accessor>().access(::cuda::std::declval<__data_handle_type>(), 0));
+
+  static constexpr bool __is_offset_noexcept =
+    noexcept(::cuda::std::declval<_Accessor>().offset(::cuda::std::declval<__data_handle_type>(), 0));
+
+  [[nodiscard]] _CCCL_API static constexpr bool __is_managed_pointer([[maybe_unused]] __data_handle_type __p) noexcept
+  {
+#if _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
+    if constexpr (::cuda::std::contiguous_iterator<__data_handle_type>)
+    {
+      return ::cuda::__is_managed_nothrow(::cuda::std::to_address(__p));
+    }
+    else
+#endif // _CCCL_HAS_CTK() && !_CCCL_COMPILER(NVRTC)
+    {
+      return true; // cannot be verified
+    }
+  }
+
+public:
+  using offset_policy    = __managed_accessor<typename _Accessor::offset_policy>;
+  using data_handle_type = __data_handle_type;
+  using reference        = typename _Accessor::reference;
+  using element_type     = typename _Accessor::element_type;
+
+  _CCCL_TEMPLATE(class _Accessor2 = _Accessor)
+  _CCCL_REQUIRES(::cuda::std::is_default_constructible_v<_Accessor2>)
+  _CCCL_API constexpr __managed_accessor() noexcept(::cuda::std::is_nothrow_default_constructible_v<_Accessor2>)
+      : _Accessor{}
+  {}
+
+  _CCCL_API constexpr __managed_accessor(_Accessor&& __acc) noexcept(
+    ::cuda::std::is_nothrow_move_constructible_v<_Accessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {}
+
+  _CCCL_API constexpr __managed_accessor(const _Accessor& __acc) noexcept(
+    ::cuda::std::is_nothrow_copy_constructible_v<_Accessor>)
+      : _Accessor{__acc}
+  {}
+
+  template <typename _OtherAccessor>
+  __managed_accessor(const __host_accessor<_OtherAccessor>&) = delete;
+
+  template <typename _OtherAccessor>
+  __managed_accessor(const __device_accessor<_OtherAccessor>&) = delete;
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    ::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr __managed_accessor(const __managed_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    !::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr explicit __managed_accessor(const __managed_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, _OtherAccessor> _CCCL_AND(
+    ::cuda::std::is_convertible_v<_OtherAccessor, _Accessor>))
+  _CCCL_API constexpr __managed_accessor(__managed_accessor<_OtherAccessor>&& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, _OtherAccessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, _OtherAccessor> _CCCL_AND(
+    !::cuda::std::is_convertible_v<_OtherAccessor, _Accessor>))
+  _CCCL_API constexpr explicit __managed_accessor(__managed_accessor<_OtherAccessor>&& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, _OtherAccessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {}
+
+  _CCCL_API constexpr reference access(data_handle_type __p, ::cuda::std::size_t __i) const
+    noexcept(__is_access_noexcept)
+  {
+    return _Accessor::access(__p, __i);
+  }
+
+  [[nodiscard]] _CCCL_API constexpr data_handle_type offset(data_handle_type __p, ::cuda::std::size_t __i) const
+    noexcept(__is_offset_noexcept)
+  {
+    return _Accessor::offset(__p, __i);
+  }
+
+#if !defined(_CCCL_DISABLE_MDSPAN_ACCESSOR_DETECT_INVALIDITY)
+  [[nodiscard]] _CCCL_API constexpr bool
+  __detectably_invalid([[maybe_unused]] data_handle_type __p, ::cuda::std::size_t) const noexcept
+  {
+    _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+    {
+      bool __is_valid = true;
+      NV_IF_ELSE_TARGET(NV_IS_HOST, (__is_valid = __is_managed_pointer(__p);), (return true;))
+      _CCCL_ASSERT(__is_valid, "managed_accessor (mdspan): data handle doesn't point to a valid managed memory");
+      return !__is_valid;
+    }
+    return false;
+  }
+#endif // !defined(_CCCL_DISABLE_MDSPAN_ACCESSOR_DETECT_INVALIDITY)
+};
+
+/***********************************************************************************************************************
+ * Accessibility Traits
+ **********************************************************************************************************************/
+
+template <typename>
+inline constexpr bool is_host_accessible_v = false;
+
+template <typename>
+inline constexpr bool is_device_accessible_v = false;
+
+template <typename _Accessor>
+inline constexpr bool is_host_accessible_v<__host_accessor<_Accessor>> = true;
+
+template <typename _Accessor>
+inline constexpr bool is_host_accessible_v<__managed_accessor<_Accessor>> = true;
+
+template <template <typename> class _TClass, typename _Accessor>
+inline constexpr bool is_host_accessible_v<_TClass<_Accessor>> = is_host_accessible_v<_Accessor>;
+
+template <typename _Accessor>
+inline constexpr bool is_device_accessible_v<__device_accessor<_Accessor>> = true;
+
+template <typename _Accessor>
+inline constexpr bool is_device_accessible_v<__managed_accessor<_Accessor>> = true;
+
+template <template <typename> class _TClass, typename _Accessor>
+inline constexpr bool is_device_accessible_v<_TClass<_Accessor>> = is_device_accessible_v<_Accessor>;
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MDSPAN_HOST_DEVICE_ACCESSOR_H

cccl_upstream/libcudacxx/include/cuda/__mdspan/host_device_mdspan.h

@@ -0,0 +1,243 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MDSPAN_HOST_DEVICE_MDSPAN_H
+#define _CUDA___MDSPAN_HOST_DEVICE_MDSPAN_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__mdspan/host_device_accessor.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__fwd/array.h>
+#include <cuda/std/__fwd/span.h>
+#include <cuda/std/__type_traits/extent.h>
+#include <cuda/std/__type_traits/is_convertible.h>
+#include <cuda/std/__type_traits/is_pointer.h>
+#include <cuda/std/__type_traits/rank.h>
+#include <cuda/std/__type_traits/remove_all_extents.h>
+#include <cuda/std/__type_traits/remove_pointer.h>
+#include <cuda/std/__type_traits/remove_reference.h>
+#include <cuda/std/__utility/delegate_constructors.h>
+#include <cuda/std/mdspan>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _ElementType,
+          typename _Extents,
+          typename _LayoutPolicy   = ::cuda::std::layout_right,
+          typename _AccessorPolicy = ::cuda::std::default_accessor<_ElementType>>
+class host_mdspan : public ::cuda::std::mdspan<_ElementType, _Extents, _LayoutPolicy, host_accessor<_AccessorPolicy>>
+{
+public:
+  _CCCL_DELEGATE_CONSTRUCTORS(
+    host_mdspan, ::cuda::std::mdspan, _ElementType, _Extents, _LayoutPolicy, host_accessor<_AccessorPolicy>);
+
+  _CCCL_API friend constexpr void swap(host_mdspan& __x, host_mdspan& __y) noexcept
+  {
+    swap(static_cast<__base&>(__x), static_cast<__base&>(__y));
+  }
+};
+
+_CCCL_TEMPLATE(class _ElementType, class... _OtherIndexTypes)
+_CCCL_REQUIRES((sizeof...(_OtherIndexTypes) > 0)
+                 _CCCL_AND(::cuda::std::is_convertible_v<_OtherIndexTypes, size_t>&&...))
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES explicit host_mdspan(_ElementType*, _OtherIndexTypes...)
+  -> host_mdspan<_ElementType, ::cuda::std::extents<size_t, ::cuda::std::__maybe_static_ext<_OtherIndexTypes>...>>;
+
+_CCCL_TEMPLATE(class _Pointer)
+_CCCL_REQUIRES(::cuda::std::is_pointer_v<::cuda::std::remove_reference_t<_Pointer>>)
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES host_mdspan(_Pointer&&)
+  -> host_mdspan<::cuda::std::remove_pointer_t<::cuda::std::remove_reference_t<_Pointer>>, ::cuda::std::extents<size_t>>;
+
+_CCCL_TEMPLATE(class _CArray)
+_CCCL_REQUIRES(::cuda::std::is_array_v<_CArray> _CCCL_AND(::cuda::std::rank_v<_CArray> == 1))
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES host_mdspan(_CArray&)
+  -> host_mdspan<::cuda::std::remove_all_extents_t<_CArray>,
+                 ::cuda::std::extents<size_t, ::cuda::std::extent_v<_CArray, 0>>>;
+
+template <class _ElementType, class _OtherIndexType, size_t _Size>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES host_mdspan(_ElementType*, const ::cuda::std::array<_OtherIndexType, _Size>&)
+  -> host_mdspan<_ElementType, ::cuda::std::dextents<size_t, _Size>>;
+
+template <class _ElementType, class _OtherIndexType, size_t _Size>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES host_mdspan(_ElementType*, ::cuda::std::span<_OtherIndexType, _Size>)
+  -> host_mdspan<_ElementType, ::cuda::std::dextents<size_t, _Size>>;
+
+// This one is necessary because all the constructors take `data_handle_type`s, not
+// `_ElementType*`s, and `data_handle_type` is taken from `accessor_type::data_handle_type`, which
+// seems to throw off automatic deduction guides.
+template <class _ElementType, class _OtherIndexType, size_t... _ExtentsPack>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES
+host_mdspan(_ElementType*, const ::cuda::std::extents<_OtherIndexType, _ExtentsPack...>&)
+  -> host_mdspan<_ElementType, ::cuda::std::extents<_OtherIndexType, _ExtentsPack...>>;
+
+template <class _ElementType, class _MappingType>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES host_mdspan(_ElementType*, const _MappingType&)
+  -> host_mdspan<_ElementType, typename _MappingType::extents_type, typename _MappingType::layout_type>;
+
+template <class _MappingType, class _AccessorType>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES
+host_mdspan(const typename _AccessorType::data_handle_type, const _MappingType&, const _AccessorType&)
+  -> host_mdspan<typename _AccessorType::element_type,
+                 typename _MappingType::extents_type,
+                 typename _MappingType::layout_type,
+                 _AccessorType>;
+
+template <typename _ElementType,
+          typename _Extents,
+          typename _LayoutPolicy   = ::cuda::std::layout_right,
+          typename _AccessorPolicy = ::cuda::std::default_accessor<_ElementType>>
+class device_mdspan
+    : public ::cuda::std::mdspan<_ElementType, _Extents, _LayoutPolicy, device_accessor<_AccessorPolicy>>
+{
+public:
+  _CCCL_DELEGATE_CONSTRUCTORS(
+    device_mdspan, ::cuda::std::mdspan, _ElementType, _Extents, _LayoutPolicy, device_accessor<_AccessorPolicy>);
+
+  _CCCL_API friend constexpr void swap(device_mdspan& __x, device_mdspan& __y) noexcept
+  {
+    swap(static_cast<__base&>(__x), static_cast<__base&>(__y));
+  }
+};
+
+_CCCL_TEMPLATE(class _ElementType, class... _OtherIndexTypes)
+_CCCL_REQUIRES((sizeof...(_OtherIndexTypes) > 0)
+                 _CCCL_AND(::cuda::std::is_convertible_v<_OtherIndexTypes, size_t>&&... && true))
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES explicit device_mdspan(_ElementType*, _OtherIndexTypes...)
+  -> device_mdspan<_ElementType, ::cuda::std::extents<size_t, ::cuda::std::__maybe_static_ext<_OtherIndexTypes>...>>;
+
+_CCCL_TEMPLATE(class _Pointer)
+_CCCL_REQUIRES(::cuda::std::is_pointer_v<::cuda::std::remove_reference_t<_Pointer>>)
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES device_mdspan(_Pointer&&)
+  -> device_mdspan<::cuda::std::remove_pointer_t<::cuda::std::remove_reference_t<_Pointer>>,
+                   ::cuda::std::extents<size_t>>;
+
+_CCCL_TEMPLATE(class _CArray)
+_CCCL_REQUIRES(::cuda::std::is_array_v<_CArray> _CCCL_AND(::cuda::std::rank_v<_CArray> == 1))
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES device_mdspan(_CArray&)
+  -> device_mdspan<::cuda::std::remove_all_extents_t<_CArray>,
+                   ::cuda::std::extents<size_t, ::cuda::std::extent_v<_CArray, 0>>>;
+
+template <class _ElementType, class _OtherIndexType, size_t _Size>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES device_mdspan(_ElementType*, const ::cuda::std::array<_OtherIndexType, _Size>&)
+  -> device_mdspan<_ElementType, ::cuda::std::dextents<size_t, _Size>>;
+
+template <class _ElementType, class _OtherIndexType, size_t _Size>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES device_mdspan(_ElementType*, ::cuda::std::span<_OtherIndexType, _Size>)
+  -> device_mdspan<_ElementType, ::cuda::std::dextents<size_t, _Size>>;
+
+// This one is necessary because all the constructors take `data_handle_type`s, not
+// `_ElementType*`s, and `data_handle_type` is taken from `accessor_type::data_handle_type`, which
+// seems to throw off automatic deduction guides.
+template <class _ElementType, class _OtherIndexType, size_t... _ExtentsPack>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES
+device_mdspan(_ElementType*, const ::cuda::std::extents<_OtherIndexType, _ExtentsPack...>&)
+  -> device_mdspan<_ElementType, ::cuda::std::extents<_OtherIndexType, _ExtentsPack...>>;
+
+template <class _ElementType, class _MappingType>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES device_mdspan(_ElementType*, const _MappingType&)
+  -> device_mdspan<_ElementType, typename _MappingType::extents_type, typename _MappingType::layout_type>;
+
+template <class _MappingType, class _AccessorType>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES
+device_mdspan(const typename _AccessorType::data_handle_type, const _MappingType&, const _AccessorType&)
+  -> device_mdspan<typename _AccessorType::element_type,
+                   typename _MappingType::extents_type,
+                   typename _MappingType::layout_type,
+                   _AccessorType>;
+
+template <typename _ElementType,
+          typename _Extents,
+          typename _LayoutPolicy   = ::cuda::std::layout_right,
+          typename _AccessorPolicy = ::cuda::std::default_accessor<_ElementType>>
+class managed_mdspan
+    : public ::cuda::std::mdspan<_ElementType, _Extents, _LayoutPolicy, managed_accessor<_AccessorPolicy>>
+{
+public:
+  _CCCL_DELEGATE_CONSTRUCTORS(
+    managed_mdspan, ::cuda::std::mdspan, _ElementType, _Extents, _LayoutPolicy, managed_accessor<_AccessorPolicy>);
+
+  _CCCL_API friend constexpr void swap(managed_mdspan& __x, managed_mdspan& __y) noexcept
+  {
+    swap(static_cast<__base&>(__x), static_cast<__base&>(__y));
+  }
+};
+
+_CCCL_TEMPLATE(class _ElementType, class... _OtherIndexTypes)
+_CCCL_REQUIRES((sizeof...(_OtherIndexTypes) > 0)
+                 _CCCL_AND(::cuda::std::is_convertible_v<_OtherIndexTypes, size_t>&&... && true))
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES explicit managed_mdspan(_ElementType*, _OtherIndexTypes...)
+  -> managed_mdspan<_ElementType, ::cuda::std::extents<size_t, ::cuda::std::__maybe_static_ext<_OtherIndexTypes>...>>;
+
+_CCCL_TEMPLATE(class _Pointer)
+_CCCL_REQUIRES(::cuda::std::is_pointer_v<::cuda::std::remove_reference_t<_Pointer>>)
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES managed_mdspan(_Pointer&&)
+  -> managed_mdspan<::cuda::std::remove_pointer_t<::cuda::std::remove_reference_t<_Pointer>>,
+                    ::cuda::std::extents<size_t>>;
+
+_CCCL_TEMPLATE(class _CArray)
+_CCCL_REQUIRES(::cuda::std::is_array_v<_CArray> _CCCL_AND(::cuda::std::rank_v<_CArray> == 1))
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES managed_mdspan(_CArray&)
+  -> managed_mdspan<::cuda::std::remove_all_extents_t<_CArray>,
+                    ::cuda::std::extents<size_t, ::cuda::std::extent_v<_CArray, 0>>>;
+
+template <class _ElementType, class _OtherIndexType, size_t _Size>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES managed_mdspan(_ElementType*, const ::cuda::std::array<_OtherIndexType, _Size>&)
+  -> managed_mdspan<_ElementType, ::cuda::std::dextents<size_t, _Size>>;
+
+template <class _ElementType, class _OtherIndexType, size_t _Size>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES managed_mdspan(_ElementType*, ::cuda::std::span<_OtherIndexType, _Size>)
+  -> managed_mdspan<_ElementType, ::cuda::std::dextents<size_t, _Size>>;
+
+// This one is necessary because all the constructors take `data_handle_type`s, not
+// `_ElementType*`s, and `data_handle_type` is taken from `accessor_type::data_handle_type`, which
+// seems to throw off automatic deduction guides.
+template <class _ElementType, class _OtherIndexType, size_t... _ExtentsPack>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES
+managed_mdspan(_ElementType*, const ::cuda::std::extents<_OtherIndexType, _ExtentsPack...>&)
+  -> managed_mdspan<_ElementType, ::cuda::std::extents<_OtherIndexType, _ExtentsPack...>>;
+
+template <class _ElementType, class _MappingType>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES managed_mdspan(_ElementType*, const _MappingType&)
+  -> managed_mdspan<_ElementType, typename _MappingType::extents_type, typename _MappingType::layout_type>;
+
+template <class _MappingType, class _AccessorType>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES
+managed_mdspan(const typename _AccessorType::data_handle_type, const _MappingType&, const _AccessorType&)
+  -> managed_mdspan<typename _AccessorType::element_type,
+                    typename _MappingType::extents_type,
+                    typename _MappingType::layout_type,
+                    _AccessorType>;
+
+/***********************************************************************************************************************
+ * Accessibility Traits
+ **********************************************************************************************************************/
+
+template <typename _Tp, typename _Ep, typename _Lp, typename _Ap>
+inline constexpr bool is_host_accessible_v<::cuda::std::mdspan<_Tp, _Ep, _Lp, _Ap>> = is_host_accessible_v<_Ap>;
+
+template <typename _Tp, typename _Ep, typename _Lp, typename _Ap>
+inline constexpr bool is_device_accessible_v<::cuda::std::mdspan<_Tp, _Ep, _Lp, _Ap>> = is_device_accessible_v<_Ap>;
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MDSPAN_HOST_DEVICE_MDSPAN_H

cccl_upstream/libcudacxx/include/cuda/__mdspan/layout_stride_relaxed.h

@@ -0,0 +1,475 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MDSPAN_LAYOUT_STRIDE_RELAXED_H
+#define _CUDA___MDSPAN_LAYOUT_STRIDE_RELAXED_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__cmath/uabs.h>
+#include <cuda/__fwd/mdspan.h>
+#include <cuda/__numeric/add_overflow.h>
+#include <cuda/__numeric/mul_overflow.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__mdspan/concepts.h>
+#include <cuda/std/__mdspan/empty_base.h>
+#include <cuda/std/__mdspan/extents.h>
+#include <cuda/std/__mdspan/submdspan_helper.h>
+#include <cuda/std/__type_traits/conjunction.h>
+#include <cuda/std/__type_traits/integral_constant_like.h>
+#include <cuda/std/__type_traits/is_constructible.h>
+#include <cuda/std/__type_traits/is_convertible.h>
+#include <cuda/std/__type_traits/is_integer.h>
+#include <cuda/std/__type_traits/is_nothrow_constructible.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/__utility/cmp.h>
+#include <cuda/std/__utility/integer_sequence.h>
+#include <cuda/std/array>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+//! @brief Computes the effective offset of a strided mapping by calling mapping(0, 0, ...).
+//! Standard layouts (layout_left, layout_right, layout_stride) don't have an explicit offset() member,
+//! but layout_stride_relaxed does. This helper enables comparing offsets in operator==.
+template <class _StridedMapping, ::cuda::std::size_t... _Pos>
+[[nodiscard]] _CCCL_API constexpr auto
+__layout_stride_relaxed_compute_offset(const _StridedMapping& __mapping, ::cuda::std::index_sequence<_Pos...>) noexcept
+{
+  using __index_type = typename _StridedMapping::index_type;
+  return static_cast<__index_type>(__mapping((static_cast<void>(_Pos), __index_type{0})...));
+}
+
+//! @brief Overload that handles rank-0 and zero-extent cases before delegating to the index_sequence version.
+template <class _StridedMapping>
+[[nodiscard]] _CCCL_API constexpr auto __layout_stride_relaxed_compute_offset(const _StridedMapping& __mapping) noexcept
+{
+  using _Extents        = typename _StridedMapping::extents_type;
+  using _RankType       = typename _StridedMapping::rank_type;
+  constexpr auto __rank = _Extents::rank();
+  // Check if any extent is zero - can't call mapping(0,...) in that case
+  bool __extent_is_zero = false;
+  for (_RankType __r = 0; __r != __rank; ++__r)
+  {
+    if (__mapping.extents().extent(__r) == 0)
+    {
+      __extent_is_zero = true;
+      break;
+    }
+  }
+  if (__extent_is_zero)
+  {
+    return typename _StridedMapping::index_type{0};
+  }
+  return ::cuda::__layout_stride_relaxed_compute_offset(__mapping, ::cuda::std::make_index_sequence<__rank>{});
+}
+
+template <class _StridedLayoutMapping, class _Extents>
+_CCCL_CONCEPT __layout_stride_relaxed_can_convert_from_strided = _CCCL_REQUIRES_EXPR((_StridedLayoutMapping, _Extents))(
+  requires(::cuda::std::__mdspan_detail::__layout_mapping_alike<_StridedLayoutMapping>),
+  requires(_StridedLayoutMapping::is_always_unique()),
+  requires(_StridedLayoutMapping::is_always_strided()),
+  requires(::cuda::std::is_constructible_v<_Extents, typename _StridedLayoutMapping::extents_type>));
+
+template <class _StridedLayoutMapping, class _Extents>
+_CCCL_CONCEPT __layout_stride_relaxed_converts_implicit_from_strided =
+  _CCCL_REQUIRES_EXPR((_StridedLayoutMapping, _Extents))(
+    requires(::cuda::std::is_convertible_v<typename _StridedLayoutMapping::extents_type, _Extents>),
+    requires(::cuda::std::__mdspan_detail::__is_mapping_of<::cuda::std::layout_left, _StridedLayoutMapping>
+             || ::cuda::std::__mdspan_detail::__is_mapping_of<::cuda::std::layout_right, _StridedLayoutMapping>
+             || ::cuda::std::__mdspan_detail::__is_mapping_of<::cuda::std::layout_stride, _StridedLayoutMapping>));
+
+/***********************************************************************************************************************
+ * layout_stride_relaxed::mapping
+ **********************************************************************************************************************/
+
+template <class _Extents, class _Strides, class _OffsetType>
+class _CCCL_DECLSPEC_EMPTY_BASES layout_stride_relaxed::mapping
+    : private ::cuda::std::__mdspan_ebco<_Extents, _Strides, _OffsetType>
+{
+public:
+  static constexpr bool __is_constant_offset = ::cuda::std::__integral_constant_like<_OffsetType>;
+  static_assert(::cuda::std::__is_cuda_std_extents_v<_Extents>,
+                "layout_stride_relaxed::mapping template argument must be a specialization of extents.");
+  static_assert(::cuda::__is_cuda_strides_v<_Strides>,
+                "layout_stride_relaxed::mapping strides template argument must be a specialization of strides.");
+  static_assert(__is_constant_offset || ::cuda::std::__cccl_is_integer_v<_OffsetType>,
+                "layout_stride_relaxed::mapping offset type must be an integer orinteger-constant-like type");
+  static_assert(_Extents::rank() == _Strides::rank(),
+                "layout_stride_relaxed::mapping: extents and strides must have the same rank");
+
+  using extents_type = _Extents;
+  using strides_type = _Strides;
+  using index_type   = typename extents_type::index_type;
+  using size_type    = typename extents_type::size_type;
+  using rank_type    = typename extents_type::rank_type;
+  using offset_type  = decltype(::cuda::std::__de_ice(_OffsetType{}));
+  using layout_type  = layout_stride_relaxed;
+  using __base       = ::cuda::std::__mdspan_ebco<_Extents, _Strides, _OffsetType>;
+
+private:
+  static constexpr rank_type __rank_    = extents_type::rank();
+  static constexpr auto __rank_sequence = ::cuda::std::make_index_sequence<extents_type::rank()>();
+
+  //! @brief Helper to construct strides from another mapping using stride(r) calls
+  template <class _StridedLayoutMapping>
+  [[nodiscard]] _CCCL_API static constexpr strides_type __make_strides(const _StridedLayoutMapping& __other) noexcept
+  {
+    ::cuda::std::array<offset_type, __rank_> __init_strides{};
+    if constexpr (__rank_ > 0)
+    {
+      for (rank_type __d = 0; __d < __rank_; ++__d)
+      {
+        _CCCL_ASSERT(::cuda::std::in_range<offset_type>(__other.stride(__d)),
+                     "layout_stride_relaxed::mapping: stride is out of range");
+        __init_strides[__d] = static_cast<offset_type>(__other.stride(__d));
+      }
+    }
+    return strides_type(__init_strides);
+  }
+
+  [[nodiscard]] _CCCL_API constexpr bool __has_positive_strides() const noexcept
+  {
+    bool __result = true;
+    for (rank_type __r = 0; __r != __rank_; ++__r)
+    {
+      if (strides().stride(__r) <= 0)
+      {
+        __result = false;
+        break;
+      }
+    }
+    return __result;
+  }
+
+public:
+  //! @brief Default constructor delegates to converting constructor from layout_right
+  _CCCL_API constexpr mapping() noexcept
+      : mapping(::cuda::std::layout_right::mapping<extents_type>{})
+  {}
+
+  _CCCL_HIDE_FROM_ABI constexpr mapping(const mapping&) noexcept            = default;
+  _CCCL_HIDE_FROM_ABI constexpr mapping& operator=(const mapping&) noexcept = default;
+
+  _CCCL_API constexpr mapping(
+    const extents_type& __ext, const strides_type& __strides, _OffsetType __offset = _OffsetType{}) noexcept
+      : __base(__ext, __strides, __offset)
+  {
+    _CCCL_ASSERT(::cuda::std::cmp_greater_equal(static_cast<offset_type>(offset()), offset_type{0}),
+                 "layout_stride_relaxed::mapping: offset must be nonnegative");
+    // not catching this could lead to out-of-bounds access later when used inside mdspan
+    // using ext_t = dextents<char, 2>;
+    // mapping<ext_t> map(ext_t(40,40));
+    // map(10, 3) == -126
+    _CCCL_ASSERT((static_cast<void>(required_span_size()), true),
+                 "layout_stride_relaxed::mapping extents ctor: required_span_size() is not representable");
+  }
+
+  //! @brief (non-explicit) Converting constructor from another layout_stride_relaxed::mapping
+  _CCCL_TEMPLATE(class _OtherMapping)
+  _CCCL_REQUIRES(::cuda::std::__mdspan_detail::__layout_mapping_alike<_OtherMapping>
+                   _CCCL_AND ::cuda::std::is_constructible_v<extents_type, typename _OtherMapping::extents_type>
+                     _CCCL_AND ::cuda::std::is_same_v<typename _OtherMapping::layout_type, layout_stride_relaxed> //
+                       _CCCL_AND(::cuda::std::is_convertible_v<typename _OtherMapping::extents_type, extents_type>))
+  _CCCL_API constexpr mapping(const _OtherMapping& __other) noexcept
+      : __base(__other.extents(), __other.strides(), __other.offset())
+  {
+    _CCCL_ASSERT(::cuda::std::cmp_greater_equal(static_cast<offset_type>(offset()), offset_type{0}),
+                 "layout_stride_relaxed::mapping: offset must be nonnegative");
+    _CCCL_ASSERT((static_cast<void>(required_span_size()), true),
+                 "layout_stride_relaxed::mapping converting ctor: required_span_size() is not representable");
+  }
+
+  //! @brief (explicit) Converting constructor from another layout_stride_relaxed::mapping
+  _CCCL_TEMPLATE(class _OtherMapping)
+  _CCCL_REQUIRES(::cuda::std::__mdspan_detail::__layout_mapping_alike<_OtherMapping>
+                   _CCCL_AND ::cuda::std::is_constructible_v<extents_type, typename _OtherMapping::extents_type>
+                     _CCCL_AND ::cuda::std::is_same_v<typename _OtherMapping::layout_type, layout_stride_relaxed> //
+                       _CCCL_AND(!::cuda::std::is_convertible_v<typename _OtherMapping::extents_type, extents_type>))
+  _CCCL_API explicit constexpr mapping(const _OtherMapping& __other) noexcept
+      : __base(__other.extents(), __other.strides(), __other.offset())
+  {
+    _CCCL_ASSERT(::cuda::std::cmp_greater_equal(static_cast<offset_type>(offset()), offset_type{0}),
+                 "layout_stride_relaxed::mapping: offset must be nonnegative");
+    _CCCL_ASSERT((static_cast<void>(required_span_size()), true),
+                 "layout_stride_relaxed::mapping converting ctor: required_span_size() is not representable");
+  }
+
+  //! @brief (non-explicit) Converting constructor from a strided layout mapping (NOT layout_stride_relaxed)
+  _CCCL_TEMPLATE(class _StridedLayoutMapping)
+  _CCCL_REQUIRES(__layout_stride_relaxed_can_convert_from_strided<_StridedLayoutMapping, extents_type> _CCCL_AND
+                   __layout_stride_relaxed_converts_implicit_from_strided<_StridedLayoutMapping, extents_type>)
+  _CCCL_API constexpr mapping(const _StridedLayoutMapping& __other) noexcept
+      : __base(__other.extents(), __make_strides(__other))
+  {
+    _CCCL_ASSERT((static_cast<void>(required_span_size()), true),
+                 "layout_stride_relaxed::mapping strided ctor: required_span_size() is not representable");
+  }
+
+  //! @brief Explicit converting constructor from a strided layout mapping (NOT layout_stride_relaxed)
+  _CCCL_TEMPLATE(class _StridedLayoutMapping)
+  _CCCL_REQUIRES(__layout_stride_relaxed_can_convert_from_strided<_StridedLayoutMapping, extents_type> _CCCL_AND(
+    !__layout_stride_relaxed_converts_implicit_from_strided<_StridedLayoutMapping, extents_type>))
+  _CCCL_API explicit constexpr mapping(const _StridedLayoutMapping& __other) noexcept
+      : __base(__other.extents(), __make_strides(__other))
+  {
+    _CCCL_ASSERT((static_cast<void>(required_span_size()), true),
+                 "layout_stride_relaxed::mapping strided ctor: required_span_size() is not representable");
+  }
+
+  // [mdspan.layout.stride.obs], observers
+
+  [[nodiscard]] _CCCL_API constexpr const extents_type& extents() const noexcept
+  {
+    return this->template __get<0>();
+  }
+
+  [[nodiscard]] _CCCL_API constexpr const strides_type& strides() const noexcept
+  {
+    return this->template __get<1>();
+  }
+
+  [[nodiscard]] _CCCL_API constexpr offset_type offset() const noexcept
+  {
+    return this->template __get<2>();
+  }
+
+  //! @brief Returns the required span size to cover all valid indices
+  [[nodiscard]] _CCCL_API constexpr index_type required_span_size() const noexcept
+  {
+    const auto __offset_val = static_cast<offset_type>(offset());
+    if constexpr (__rank_ == 0)
+    {
+      // For rank-0 mappings, there is exactly one valid index.
+      // The required span size is the maximum mapped index + 1.
+      _CCCL_ASSERT(!::cuda::add_overflow<index_type>(__offset_val, offset_type{1}),
+                   "layout_stride_relaxed::mapping: required_span_size is not representable as index_type");
+      return static_cast<index_type>(__offset_val + offset_type{1});
+    }
+    else
+    {
+      // The dot product of indices and strides is linear.
+      // Thus, over all valid indices, the max value of the dot product is achieved at the extrema: either the min
+      // index (0) if the stride is negative, or the max index (extent(r) - 1) if the stride is non-negative.
+      // For non-negative stride: max contribution is (extent - 1) * stride, min contribution is 0
+      // For negative stride: max contribution is 0 (max achieved at index 0), min is -(extent - 1) * |stride|
+      // __min_dot tracks the total positive magnitude of the negative contributions
+      index_type __dot{1};
+      offset_type __min_dot{0};
+      bool __stride_is_negative = false;
+      for (rank_type __r = 0; __r < __rank_; ++__r)
+      {
+        const auto __ext = extents().extent(__r);
+        if (__ext == index_type{0})
+        {
+          __stride_is_negative = true;
+          break;
+        }
+        const auto __stride_val = strides().stride(__r);
+        _CCCL_ASSERT(::cuda::std::in_range<index_type>(::cuda::uabs(__stride_val)),
+                     "layout_stride_relaxed::mapping: stride is out of range");
+        if (__stride_val < 0)
+        {
+          _CCCL_ASSERT(::cuda::std::in_range<offset_type>(__ext - 1),
+                       "layout_stride_relaxed::mapping: extent - 1 is not representable as offset_type");
+          const auto __min_extent   = static_cast<offset_type>(__ext - 1);
+          const auto __abs_stride_u = ::cuda::uabs(__stride_val);
+          _CCCL_ASSERT(::cuda::std::in_range<offset_type>(__abs_stride_u),
+                       "layout_stride_relaxed::mapping: absolute stride is not representable as offset_type");
+          const auto __abs_stride = static_cast<offset_type>(__abs_stride_u);
+          _CCCL_ASSERT(!::cuda::mul_overflow(__min_extent, __abs_stride)
+                         && !::cuda::add_overflow(__min_extent * __abs_stride, __min_dot),
+                       "layout_stride_relaxed::mapping: minimum mapped index is not representable");
+          __min_dot += __min_extent * __abs_stride;
+        }
+        const auto __max_index = __stride_val < 0 ? index_type{0} : static_cast<index_type>(__ext - 1);
+        const auto __stride    = static_cast<index_type>(__stride_val);
+        _CCCL_ASSERT(!::cuda::mul_overflow<index_type>(__max_index, __stride)
+                       && !::cuda::add_overflow(__max_index * __stride, __dot),
+                     "layout_stride_relaxed::mapping: required_span_size is not representable as index_type");
+        __dot += __max_index * __stride;
+      }
+      _CCCL_ASSERT(::cuda::std::cmp_greater_equal(__offset_val, __min_dot),
+                   "layout_stride_relaxed::mapping: offset is insufficient for negative strides");
+      _CCCL_ASSERT(!::cuda::add_overflow<index_type>(__offset_val, __dot),
+                   "layout_stride_relaxed::mapping: required_span_size is not representable as index_type");
+      return __stride_is_negative ? index_type{0} : static_cast<index_type>(__offset_val + __dot);
+    }
+  }
+
+  template <class _Index>
+  [[nodiscard]] _CCCL_API constexpr bool __is_valid_index([[maybe_unused]] _Index __index) const noexcept
+  {
+    if constexpr (::cuda::std::__cccl_is_integer_v<_Index>)
+    {
+      return ::cuda::std::cmp_greater_equal(__index, index_type{0}) && ::cuda::std::in_range<index_type>(__index);
+    }
+    else
+    {
+      return true; // cannot be verified for custom index types
+    }
+  }
+
+  template <::cuda::std::size_t... _Pos, class... _Indices>
+  [[nodiscard]] _CCCL_API constexpr index_type
+  __compute_index(::cuda::std::index_sequence<_Pos...>, _Indices... __indices) const noexcept
+  {
+    _CCCL_ASSERT((__is_valid_index(__indices) && ...),
+                 "layout_stride_relaxed::mapping: index is not representable as index_type");
+    _CCCL_ASSERT(((static_cast<index_type>(__indices) < extents().extent(_Pos)) && ...),
+                 "layout_stride_relaxed::mapping: index is out of bounds");
+    return (static_cast<index_type>(offset()) + ...
+            + (static_cast<index_type>(__indices) * static_cast<index_type>(strides().stride(_Pos))));
+  }
+
+  //! @brief Maps multidimensional indices to a linear index
+  _CCCL_TEMPLATE(class... _Indices)
+  _CCCL_REQUIRES(
+    (sizeof...(_Indices) == __rank_) //
+    _CCCL_AND(::cuda::std::conjunction_v<::cuda::std::is_convertible<_Indices, index_type>...>)
+      _CCCL_AND(::cuda::std::conjunction_v<::cuda::std::is_nothrow_constructible<index_type, _Indices>...>))
+  [[nodiscard]] _CCCL_API constexpr index_type operator()(_Indices... __indices) const noexcept
+  {
+    return __compute_index(__rank_sequence, __indices...);
+  }
+
+  //! @brief Returns false - not always unique due to zero/negative strides
+  [[nodiscard]] _CCCL_API static constexpr bool is_always_unique() noexcept
+  {
+    return false;
+  }
+
+  //! @brief Returns false - not always exhaustive due to zero/negative strides
+  [[nodiscard]] _CCCL_API static constexpr bool is_always_exhaustive() noexcept
+  {
+    return false;
+  }
+
+  //! @brief Returns false if it is not possible to verify the offset is zero (to accommodate negative strides)
+  [[nodiscard]] _CCCL_API static constexpr bool is_always_strided() noexcept
+  {
+    if constexpr (__is_constant_offset)
+    {
+      return _OffsetType::value == 0;
+    }
+    else
+    {
+      return false;
+    }
+  }
+
+  //! @brief Returns false - uniqueness depends on strides (conservative)
+  [[nodiscard]] _CCCL_API constexpr bool is_unique() const noexcept
+  {
+    // Conservative: negative/zero strides make uniqueness hard to determine
+    return false;
+  }
+
+  //! @brief Returns false - exhaustiveness depends on strides (conservative)
+  [[nodiscard]] _CCCL_API constexpr bool is_exhaustive() const noexcept
+  {
+    // Conservative: negative/zero strides make exhaustiveness hard to determine
+    return false;
+  }
+
+  //! @brief Returns true if offset is zero (standard strided behavior)
+  [[nodiscard]] _CCCL_API constexpr bool is_strided() const noexcept
+  {
+    return static_cast<offset_type>(offset()) == offset_type{0};
+  }
+
+  [[nodiscard]] _CCCL_API constexpr offset_type stride(rank_type __r) const noexcept
+  {
+    _CCCL_ASSERT((__rank_ == 0) ? false : __r < __rank_,
+                 "layout_stride_relaxed::mapping::stride(): invalid rank index");
+    return strides().stride(__r);
+  }
+
+  // Conversion operators to layout_stride
+
+  //! @brief Explicit conversion to layout_stride::mapping
+  //! @pre offset() == 0 and all strides are positive
+  [[nodiscard]] _CCCL_API explicit constexpr operator ::cuda::std::layout_stride::mapping<extents_type>() const noexcept
+  {
+    _CCCL_ASSERT(static_cast<offset_type>(offset()) == offset_type{0},
+                 "layout_stride_relaxed::mapping: cannot convert to layout_stride with non-zero offset");
+    _CCCL_ASSERT(__has_positive_strides(),
+                 "layout_stride_relaxed::mapping: cannot convert to layout_stride with non-positive strides");
+    ::cuda::std::array<index_type, __rank_> __stride_array{};
+    for (rank_type __r = 0; __r != __rank_; ++__r)
+    {
+      __stride_array[__r] = static_cast<index_type>(strides().stride(__r));
+    }
+    return ::cuda::std::layout_stride::mapping<extents_type>(extents(), __stride_array);
+  }
+
+  // [mdspan.layout.stride.cmp], comparison
+
+  // __rhs is also a layout_stride_relaxed::mapping
+  _CCCL_TEMPLATE(class _OtherMapping)
+  _CCCL_REQUIRES(::cuda::std::__mdspan_detail::__layout_mapping_alike<_OtherMapping> //
+                   _CCCL_AND(__rank_ == _OtherMapping::extents_type::rank())
+                     _CCCL_AND(::cuda::std::is_same_v<layout_type, typename _OtherMapping::layout_type>))
+  [[nodiscard]] _CCCL_API friend constexpr bool operator==(const mapping& __lhs, const _OtherMapping& __rhs) noexcept
+  {
+    return __lhs.extents() == __rhs.extents() && ::cuda::std::cmp_equal(__lhs.offset(), __rhs.offset())
+        && __lhs.strides() == __rhs.strides();
+  }
+
+  // __rhs is NOT a layout_stride_relaxed::mapping
+  _CCCL_TEMPLATE(class _OtherMapping)
+  _CCCL_REQUIRES(::cuda::std::__mdspan_detail::__layout_mapping_alike<_OtherMapping> //
+                   _CCCL_AND(__rank_ == _OtherMapping::extents_type::rank())
+                     _CCCL_AND(_OtherMapping::is_always_strided()))
+  [[nodiscard]] _CCCL_API friend constexpr bool operator==(const mapping& __lhs, const _OtherMapping& __rhs) noexcept
+  {
+    return __lhs.extents() == __rhs.extents()
+        && ::cuda::std::cmp_equal(__lhs.offset(), ::cuda::__layout_stride_relaxed_compute_offset(__rhs))
+        && __lhs.strides() == __make_strides(__rhs);
+  }
+
+#if _CCCL_STD_VER <= 2017
+  // __rhs is also a layout_stride_relaxed::mapping
+  _CCCL_TEMPLATE(class _OtherMapping)
+  _CCCL_REQUIRES(::cuda::std::__mdspan_detail::__layout_mapping_alike<_OtherMapping> //
+                   _CCCL_AND(__rank_ == _OtherMapping::extents_type::rank())
+                     _CCCL_AND(::cuda::std::is_same_v<layout_type, typename _OtherMapping::layout_type>))
+  [[nodiscard]] _CCCL_API friend constexpr bool operator!=(const mapping& __lhs, const _OtherMapping& __rhs) noexcept
+  {
+    return !(__lhs == __rhs);
+  }
+
+  // __rhs is NOT a layout_stride_relaxed::mapping
+  _CCCL_TEMPLATE(class _OtherMapping)
+  _CCCL_REQUIRES(::cuda::std::__mdspan_detail::__layout_mapping_alike<_OtherMapping> //
+                   _CCCL_AND(__rank_ == _OtherMapping::extents_type::rank())
+                     _CCCL_AND(_OtherMapping::is_always_strided()))
+  [[nodiscard]] _CCCL_API friend constexpr bool operator!=(const mapping& __lhs, const _OtherMapping& __rhs) noexcept
+  {
+    return !(__lhs == __rhs);
+  }
+#endif // _CCCL_STD_VER <= 2017
+};
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MDSPAN_LAYOUT_STRIDE_RELAXED_H

cccl_upstream/libcudacxx/include/cuda/__mdspan/mdspan_to_dlpack.h

@@ -0,0 +1,251 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the libcu++ Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MDSPAN_MDSPAN_TO_DLPACK_H
+#define _CUDA___MDSPAN_MDSPAN_TO_DLPACK_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#if _CCCL_HAS_DLPACK()
+
+#  include <cuda/__internal/dlpack.h>
+
+#  if _CCCL_HAS_DLPACK_VERSION_1()
+#    include <cuda/__driver/driver_api.h>
+#    include <cuda/__mdspan/host_device_mdspan.h>
+#    include <cuda/__type_traits/is_floating_point.h>
+#    include <cuda/__type_traits/is_vector_type.h>
+#    include <cuda/std/__cstddef/types.h>
+#    include <cuda/std/__exception/cuda_error.h>
+#    include <cuda/std/__exception/exception_macros.h>
+#    include <cuda/std/__fwd/complex.h>
+#    include <cuda/std/__host_stdlib/stdexcept>
+#    include <cuda/std/__limits/numeric_limits.h>
+#    include <cuda/std/__type_traits/always_false.h>
+#    include <cuda/std/__type_traits/is_pointer.h>
+#    include <cuda/std/__type_traits/is_same.h>
+#    include <cuda/std/__type_traits/num_bits.h>
+#    include <cuda/std/__type_traits/remove_cv.h>
+#    include <cuda/std/__utility/cmp.h>
+#    include <cuda/std/array>
+#    include <cuda/std/cstdint>
+#    include <cuda/std/mdspan>
+
+#    include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _ElementType>
+[[nodiscard]] _CCCL_HOST_API inline ::DLDataType __data_type_to_dlpack() noexcept
+{
+  if constexpr (::cuda::std::is_same_v<_ElementType, bool>)
+  {
+    return ::DLDataType{::kDLBool, 8, 1};
+  }
+  //--------------------------------------------------------------------------------------------------------------------
+  // Signed integer types
+  else if constexpr (::cuda::std::__cccl_is_integer_v<_ElementType>)
+  {
+    return ::DLDataType{
+      (::cuda::std::is_signed_v<_ElementType>) ? ::kDLInt : ::kDLUInt, ::cuda::std::__num_bits_v<_ElementType>, 1};
+  }
+  //--------------------------------------------------------------------------------------------------------------------
+  // bfloat16 (must come before general floating-point)
+#    if _CCCL_HAS_NVBF16()
+  else if constexpr (::cuda::std::is_same_v<_ElementType, ::__nv_bfloat16>)
+  {
+    return ::DLDataType{::kDLBfloat, 16, 1};
+  }
+#    endif // _CCCL_HAS_NVBF16()
+  //--------------------------------------------------------------------------------------------------------------------
+  // Low-precision Floating-point types (must come before general floating-point)
+#    if _CCCL_HAS_NVFP8_E4M3()
+  else if constexpr (::cuda::std::is_same_v<_ElementType, ::__nv_fp8_e4m3>)
+  {
+    return ::DLDataType{::kDLFloat8_e4m3fn, 8, 1};
+  }
+#    endif // _CCCL_HAS_NVFP8_E4M3()
+#    if _CCCL_HAS_NVFP8_E5M2()
+  else if constexpr (::cuda::std::is_same_v<_ElementType, ::__nv_fp8_e5m2>)
+  {
+    return ::DLDataType{::kDLFloat8_e5m2, 8, 1};
+  }
+#    endif // _CCCL_HAS_NVFP8_E5M2()
+#    if _CCCL_HAS_NVFP8_E8M0()
+  else if constexpr (::cuda::std::is_same_v<_ElementType, ::__nv_fp8_e8m0>)
+  {
+    return ::DLDataType{::kDLFloat8_e8m0fnu, 8, 1};
+  }
+#    endif // _CCCL_HAS_NVFP8_E8M0()
+#    if _CCCL_HAS_NVFP6_E2M3()
+  else if constexpr (::cuda::std::is_same_v<_ElementType, ::__nv_fp6_e2m3>)
+  {
+    return ::DLDataType{::kDLFloat6_e2m3fn, 6, 1};
+  }
+#    endif // _CCCL_HAS_NVFP6_E2M3()
+#    if _CCCL_HAS_NVFP6_E3M2()
+  else if constexpr (::cuda::std::is_same_v<_ElementType, ::__nv_fp6_e3m2>)
+  {
+    return ::DLDataType{::kDLFloat6_e3m2fn, 6, 1};
+  }
+#    endif // _CCCL_HAS_NVFP6_E3M2()
+#    if _CCCL_HAS_NVFP4_E2M1()
+  else if constexpr (::cuda::std::is_same_v<_ElementType, ::__nv_fp4_e2m1>)
+  {
+    return ::DLDataType{::kDLFloat4_e2m1fn, 4, 1};
+  }
+#    endif // _CCCL_HAS_NVFP4_E2M1()
+  //--------------------------------------------------------------------------------------------------------------------
+  // Floating-point types (after specific types)
+  else if constexpr (::cuda::is_floating_point_v<_ElementType>)
+  {
+    return ::DLDataType{::kDLFloat, ::cuda::std::__num_bits_v<_ElementType>, 1};
+  }
+  //--------------------------------------------------------------------------------------------------------------------
+  // Complex types
+  // 256-bit data types are not supported in DLPack, e.g. cuda::std::complex<__float128>
+  else if constexpr (::cuda::std::__is_cuda_std_complex_v<_ElementType> && sizeof(_ElementType) <= sizeof(double) * 2)
+  {
+    // DLPack encodes complex numbers as a compact struct of two scalar values, and `bits` stores
+    // the size of the full complex number (e.g. std::complex<float> => bits=64).
+    return ::DLDataType{::kDLComplex, sizeof(_ElementType) * CHAR_BIT, 1};
+  }
+  //--------------------------------------------------------------------------------------------------------------------
+  // CUDA built-in vector types
+#    if _CCCL_HAS_CTK()
+  else if constexpr (::cuda::is_vector_type_v<_ElementType> || ::cuda::is_extended_fp_vector_type_v<_ElementType>)
+  {
+    constexpr ::cuda::std::uint16_t __lanes = ::cuda::std::tuple_size_v<_ElementType>;
+    if constexpr (__lanes == 2 || __lanes == 4)
+    {
+      using __scalar_t = ::cuda::std::remove_cv_t<::cuda::std::tuple_element_t<0, _ElementType>>;
+      auto __scalar    = ::cuda::__data_type_to_dlpack<__scalar_t>();
+      __scalar.lanes   = __lanes;
+      return __scalar;
+    }
+    else
+    {
+      static_assert(::cuda::std::__always_false_v<_ElementType>, "Unsupported vector type");
+      return ::DLDataType{};
+    }
+  }
+#    endif // _CCCL_HAS_CTK()
+  //--------------------------------------------------------------------------------------------------------------------
+  // Unsupported types
+  else
+  {
+    static_assert(::cuda::std::__always_false_v<_ElementType>, "Unsupported type");
+    return ::DLDataType{};
+  }
+}
+
+template <::cuda::std::size_t _Rank>
+struct __dlpack_tensor
+{
+  ::cuda::std::array<::cuda::std::int64_t, _Rank> __shape{};
+  ::cuda::std::array<::cuda::std::int64_t, _Rank> __strides{};
+  ::DLTensor __tensor{};
+
+  [[nodiscard]] _CCCL_HOST_API ::DLTensor get() const& noexcept _CCCL_LIFETIMEBOUND
+  {
+    auto __tensor1    = __tensor;
+    __tensor1.shape   = _Rank > 0 ? const_cast<::cuda::std::int64_t*>(__shape.data()) : nullptr;
+    __tensor1.strides = _Rank > 0 ? const_cast<::cuda::std::int64_t*>(__strides.data()) : nullptr;
+    return __tensor1;
+  }
+
+  ::DLTensor get() const&& = delete;
+};
+
+template <typename _ElementType, typename _Extents, typename _Layout, typename _Accessor>
+[[nodiscard]] _CCCL_HOST_API __dlpack_tensor<_Extents::rank()>
+__to_dlpack(const ::cuda::std::mdspan<_ElementType, _Extents, _Layout, _Accessor>& __mdspan,
+            ::DLDeviceType __device_type,
+            int __device_id)
+{
+  static_assert(::cuda::std::is_pointer_v<typename _Accessor::data_handle_type>, "data_handle_type must be a pointer");
+  using __element_type = ::cuda::std::remove_cv_t<_ElementType>;
+  __dlpack_tensor<_Extents::rank()> __wrapper{};
+  auto& __tensor  = __wrapper.__tensor;
+  __tensor.data   = __mdspan.size() > 0 ? const_cast<__element_type*>(__mdspan.data_handle()) : nullptr;
+  __tensor.device = ::DLDevice{__device_type, __device_id};
+  __tensor.ndim   = static_cast<int>(__mdspan.rank());
+  __tensor.dtype  = ::cuda::__data_type_to_dlpack<::cuda::std::remove_cv_t<_ElementType>>();
+  if constexpr (_Extents::rank() > 0)
+  {
+    constexpr auto __max_extent = ::cuda::std::numeric_limits<::cuda::std::int64_t>::max();
+    for (::cuda::std::size_t __i = 0; __i < __mdspan.rank(); ++__i)
+    {
+      if (::cuda::std::cmp_greater(__mdspan.extent(__i), __max_extent))
+      {
+        _CCCL_THROW(::std::invalid_argument, "Extent is too large");
+      }
+      if (::cuda::std::cmp_greater(__mdspan.stride(__i), __max_extent))
+      {
+        _CCCL_THROW(::std::invalid_argument, "Stride is too large");
+      }
+      __wrapper.__shape[__i]   = static_cast<::cuda::std::int64_t>(__mdspan.extent(__i));
+      __wrapper.__strides[__i] = static_cast<::cuda::std::int64_t>(__mdspan.stride(__i));
+    }
+  }
+  __tensor.byte_offset = 0;
+  return __wrapper;
+}
+
+/***********************************************************************************************************************
+ * Public API
+ **********************************************************************************************************************/
+
+template <typename _ElementType, typename _Extents, typename _Layout, typename _Accessor>
+[[nodiscard]] _CCCL_HOST_API __dlpack_tensor<_Extents::rank()>
+to_dlpack_tensor(const ::cuda::host_mdspan<_ElementType, _Extents, _Layout, _Accessor>& __mdspan)
+{
+  using __mdspan_type = ::cuda::std::mdspan<_ElementType, _Extents, _Layout, _Accessor>;
+  return ::cuda::__to_dlpack(__mdspan_type{__mdspan}, ::kDLCPU, 0);
+}
+
+template <typename _ElementType, typename _Extents, typename _Layout, typename _Accessor>
+[[nodiscard]] _CCCL_HOST_API __dlpack_tensor<_Extents::rank()>
+to_dlpack_tensor(const ::cuda::device_mdspan<_ElementType, _Extents, _Layout, _Accessor>& __mdspan)
+{
+  using __mdspan_type              = ::cuda::std::mdspan<_ElementType, _Extents, _Layout, _Accessor>;
+  ::CUpointer_attribute __attrs[1] = {::CU_POINTER_ATTRIBUTE_DEVICE_ORDINAL};
+  int __ptr_dev_id                 = 0;
+  void* __results[1]               = {&__ptr_dev_id};
+  const auto __status = ::cuda::__driver::__pointerGetAttributesNoThrow(__attrs, __results, __mdspan.data_handle());
+  if (__status != ::cudaSuccess)
+  {
+    _CCCL_THROW(::cuda::cuda_error, __status, "Failed to get device ordinal of a pointer");
+  }
+  return ::cuda::__to_dlpack(__mdspan_type{__mdspan}, ::kDLCUDA, __ptr_dev_id);
+}
+
+template <typename _ElementType, typename _Extents, typename _Layout, typename _Accessor>
+[[nodiscard]] _CCCL_HOST_API __dlpack_tensor<_Extents::rank()>
+to_dlpack_tensor(const ::cuda::managed_mdspan<_ElementType, _Extents, _Layout, _Accessor>& __mdspan)
+{
+  using __mdspan_type = ::cuda::std::mdspan<_ElementType, _Extents, _Layout, _Accessor>;
+  return ::cuda::__to_dlpack(__mdspan_type{__mdspan}, ::kDLCUDAManaged, 0);
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#    include <cuda/std/__cccl/epilogue.h>
+
+#  endif // _CCCL_HAS_DLPACK_VERSION_1()
+#endif // _CCCL_HAS_DLPACK()
+#endif // _CUDA___MDSPAN_MDSPAN_TO_DLPACK_H

cccl_upstream/libcudacxx/include/cuda/__mdspan/restrict_accessor.h

@@ -0,0 +1,152 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MDSPAN_RESTRICT_ACCESSOR_H
+#define _CUDA___MDSPAN_RESTRICT_ACCESSOR_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__type_traits/is_constructible.h>
+#include <cuda/std/__type_traits/is_convertible.h>
+#include <cuda/std/__type_traits/is_default_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
+#include <cuda/std/__type_traits/is_pointer.h>
+#include <cuda/std/__type_traits/remove_pointer.h>
+#include <cuda/std/__utility/declval.h>
+#include <cuda/std/__utility/move.h>
+#include <cuda/std/cstddef>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _Accessor>
+class __restrict_accessor;
+
+template <typename _Accessor>
+using restrict_accessor = __restrict_accessor<_Accessor>;
+
+/***********************************************************************************************************************
+ * Accessor Traits
+ **********************************************************************************************************************/
+
+template <typename>
+inline constexpr bool is_restrict_accessor_v = false;
+
+template <typename _Accessor>
+inline constexpr bool is_restrict_accessor_v<__restrict_accessor<_Accessor>> = true;
+
+/***********************************************************************************************************************
+ * Restrict Accessor
+ **********************************************************************************************************************/
+
+_CCCL_DIAG_PUSH
+_CCCL_DIAG_SUPPRESS_GCC("-Wignored-qualifiers")
+
+template <typename _Accessor>
+class __restrict_accessor : public _Accessor
+{
+  static_assert(::cuda::std::is_pointer_v<typename _Accessor::data_handle_type>, "Accessor must be pointer based");
+
+  using __data_handle_type = typename _Accessor::data_handle_type;
+  using __element_type     = ::cuda::std::remove_pointer_t<__data_handle_type>;
+
+  static constexpr bool __is_access_noexcept =
+    noexcept(::cuda::std::declval<_Accessor>().access(::cuda::std::declval<__data_handle_type>(), 0));
+
+  static constexpr bool __is_offset_noexcept =
+    noexcept(::cuda::std::declval<_Accessor>().offset(::cuda::std::declval<__data_handle_type>(), 0));
+
+public:
+  using offset_policy    = __restrict_accessor<typename _Accessor::offset_policy>;
+  using data_handle_type = __element_type* _CCCL_RESTRICT;
+  using reference        = typename _Accessor::reference;
+  using element_type     = typename _Accessor::element_type;
+
+  _CCCL_TEMPLATE(class _Accessor2 = _Accessor)
+  _CCCL_REQUIRES(::cuda::std::is_default_constructible_v<_Accessor2>)
+  _CCCL_API constexpr __restrict_accessor() noexcept(::cuda::std::is_nothrow_default_constructible_v<_Accessor2>)
+      : _Accessor{}
+  {}
+
+  _CCCL_API constexpr __restrict_accessor(const _Accessor& __acc) noexcept(
+    ::cuda::std::is_nothrow_copy_constructible_v<_Accessor>)
+      : _Accessor{__acc}
+  {}
+
+  _CCCL_API constexpr __restrict_accessor(_Accessor&& __acc) noexcept(
+    ::cuda::std::is_nothrow_move_constructible_v<_Accessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    ::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr __restrict_accessor(const __restrict_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    !::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr explicit __restrict_accessor(const __restrict_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, _OtherAccessor> _CCCL_AND(
+    ::cuda::std::is_convertible_v<_OtherAccessor, _Accessor>))
+  _CCCL_API constexpr __restrict_accessor(__restrict_accessor<_OtherAccessor>&& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, _OtherAccessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {}
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, _OtherAccessor> _CCCL_AND(
+    !::cuda::std::is_convertible_v<_OtherAccessor, _Accessor>))
+  _CCCL_API constexpr explicit __restrict_accessor(__restrict_accessor<_OtherAccessor>&& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, _OtherAccessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {}
+
+  _CCCL_API constexpr reference access(__element_type* _CCCL_RESTRICT __p, size_t __i) const
+    noexcept(__is_access_noexcept)
+  {
+    return _Accessor::access(__p, __i);
+  }
+
+  _CCCL_API constexpr data_handle_type offset(__element_type* _CCCL_RESTRICT __p, size_t __i) const
+    noexcept(__is_offset_noexcept)
+  {
+    return _Accessor::offset(__p, __i);
+  }
+};
+
+_CCCL_DIAG_POP
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MDSPAN_RESTRICT_ACCESSOR_H

cccl_upstream/libcudacxx/include/cuda/__mdspan/restrict_mdspan.h

@@ -0,0 +1,119 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MDSPAN_RESTRICT_MDSPAN_H
+#define _CUDA___MDSPAN_RESTRICT_MDSPAN_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__mdspan/restrict_accessor.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__fwd/array.h>
+#include <cuda/std/__fwd/span.h>
+#include <cuda/std/__type_traits/extent.h>
+#include <cuda/std/__type_traits/is_convertible.h>
+#include <cuda/std/__type_traits/is_pointer.h>
+#include <cuda/std/__type_traits/rank.h>
+#include <cuda/std/__type_traits/remove_all_extents.h>
+#include <cuda/std/__type_traits/remove_pointer.h>
+#include <cuda/std/__type_traits/remove_reference.h>
+#include <cuda/std/__utility/delegate_constructors.h>
+#include <cuda/std/mdspan>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _ElementType,
+          typename _Extents,
+          typename _LayoutPolicy   = ::cuda::std::layout_right,
+          typename _AccessorPolicy = ::cuda::std::default_accessor<_ElementType>>
+class restrict_mdspan
+    : public ::cuda::std::mdspan<_ElementType, _Extents, _LayoutPolicy, restrict_accessor<_AccessorPolicy>>
+{
+public:
+  _CCCL_DELEGATE_CONSTRUCTORS(
+    restrict_mdspan, ::cuda::std::mdspan, _ElementType, _Extents, _LayoutPolicy, restrict_accessor<_AccessorPolicy>);
+
+  _CCCL_API friend constexpr void swap(restrict_mdspan& __x, restrict_mdspan& __y) noexcept
+  {
+    swap(static_cast<__base&>(__x), static_cast<__base&>(__y));
+  }
+};
+
+_CCCL_TEMPLATE(class _ElementType, class... _OtherIndexTypes)
+_CCCL_REQUIRES((sizeof...(_OtherIndexTypes) > 0)
+                 _CCCL_AND(::cuda::std::is_convertible_v<_OtherIndexTypes, size_t>&&... && true))
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES explicit restrict_mdspan(_ElementType*, _OtherIndexTypes...)
+  -> restrict_mdspan<_ElementType, ::cuda::std::extents<size_t, ::cuda::std::__maybe_static_ext<_OtherIndexTypes>...>>;
+
+_CCCL_TEMPLATE(class _Pointer)
+_CCCL_REQUIRES(::cuda::std::is_pointer_v<::cuda::std::remove_reference_t<_Pointer>>)
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES restrict_mdspan(_Pointer&&)
+  -> restrict_mdspan<::cuda::std::remove_pointer_t<::cuda::std::remove_reference_t<_Pointer>>,
+                     ::cuda::std::extents<size_t>>;
+
+_CCCL_TEMPLATE(class _CArray)
+_CCCL_REQUIRES(::cuda::std::is_array_v<_CArray> _CCCL_AND(::cuda::std::rank_v<_CArray> == 1))
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES restrict_mdspan(_CArray&)
+  -> restrict_mdspan<::cuda::std::remove_all_extents_t<_CArray>,
+                     ::cuda::std::extents<size_t, ::cuda::std::extent_v<_CArray, 0>>>;
+
+template <class _ElementType, class _OtherIndexType, size_t _Size>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES restrict_mdspan(_ElementType*, const ::cuda::std::array<_OtherIndexType, _Size>&)
+  -> restrict_mdspan<_ElementType, ::cuda::std::dextents<size_t, _Size>>;
+
+template <class _ElementType, class _OtherIndexType, size_t _Size>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES restrict_mdspan(_ElementType*, ::cuda::std::span<_OtherIndexType, _Size>)
+  -> restrict_mdspan<_ElementType, ::cuda::std::dextents<size_t, _Size>>;
+
+// This one is necessary because all the constructors take `data_handle_type`s, not
+// `_ElementType*`s, and `data_handle_type` is taken from `accessor_type::data_handle_type`, which
+// seems to throw off automatic deduction guides.
+template <class _ElementType, class _OtherIndexType, size_t... _ExtentsPack>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES
+restrict_mdspan(_ElementType*, const ::cuda::std::extents<_OtherIndexType, _ExtentsPack...>&)
+  -> restrict_mdspan<_ElementType, ::cuda::std::extents<_OtherIndexType, _ExtentsPack...>>;
+
+template <class _ElementType, class _MappingType>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES restrict_mdspan(_ElementType*, const _MappingType&)
+  -> restrict_mdspan<_ElementType, typename _MappingType::extents_type, typename _MappingType::layout_type>;
+
+template <class _MappingType, class _AccessorType>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES
+restrict_mdspan(const typename _AccessorType::data_handle_type, const _MappingType&, const _AccessorType&)
+  -> restrict_mdspan<typename _AccessorType::element_type,
+                     typename _MappingType::extents_type,
+                     typename _MappingType::layout_type,
+                     _AccessorType>;
+
+/***********************************************************************************************************************
+ * Accessibility Traits
+ **********************************************************************************************************************/
+
+template <typename>
+inline constexpr bool is_restrict_mdspan_v = false;
+
+template <typename _Tp, typename _Ep, typename _Lp, typename _Ap>
+inline constexpr bool is_restrict_mdspan_v<::cuda::std::mdspan<_Tp, _Ep, _Lp, _Ap>> = is_restrict_accessor_v<_Ap>;
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MDSPAN_RESTRICT_MDSPAN_H

cccl_upstream/libcudacxx/include/cuda/__mdspan/shared_memory_accessor.h

@@ -0,0 +1,212 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MDSPAN_SHARED_MEMORY_ACCESSOR_H
+#define _CUDA___MDSPAN_SHARED_MEMORY_ACCESSOR_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__memory/address_space.h>
+#include <cuda/__ptx/instructions/get_sreg.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__type_traits/is_constructible.h>
+#include <cuda/std/__type_traits/is_convertible.h>
+#include <cuda/std/__type_traits/is_default_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_copy_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_default_constructible.h>
+#include <cuda/std/__type_traits/is_nothrow_move_constructible.h>
+#include <cuda/std/__type_traits/is_pointer.h>
+#include <cuda/std/__type_traits/remove_pointer.h>
+#include <cuda/std/__utility/declval.h>
+#include <cuda/std/__utility/move.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _Accessor>
+class __shared_memory_accessor;
+
+template <typename _Accessor>
+using shared_memory_accessor = __shared_memory_accessor<_Accessor>;
+
+/***********************************************************************************************************************
+ * Accessor Traits
+ **********************************************************************************************************************/
+
+template <typename>
+inline constexpr bool is_shared_memory_accessor_v = false;
+
+template <typename _Accessor>
+inline constexpr bool is_shared_memory_accessor_v<__shared_memory_accessor<_Accessor>> = true;
+
+#define _CCCL_VERIFY_DEVICE_ONLY_USAGE() \
+  NV_IF_TARGET(NV_IS_HOST, (_CCCL_VERIFY(false, "the function cannot be used in HOST code");))
+
+/***********************************************************************************************************************
+ * Shared Memory Accessor
+ **********************************************************************************************************************/
+
+#if _CCCL_CUDA_COMPILATION()
+
+// TODO: move to a more appropriate place
+[[nodiscard]] _CCCL_DEVICE_API inline ::cuda::std::uint32_t __max_smem_allocation_bytes() noexcept
+{
+  const auto __total_smem_size   = ::cuda::ptx::get_sreg_total_smem_size();
+  const auto __dynamic_smem_size = ::cuda::ptx::get_sreg_dynamic_smem_size();
+  const auto __static_smem_size  = __total_smem_size - __dynamic_smem_size;
+  const auto __max_smem_size     = ::max(__static_smem_size, __dynamic_smem_size);
+  return __max_smem_size;
+}
+#endif // _CCCL_CUDA_COMPILATION()
+
+template <typename _Accessor>
+class __shared_memory_accessor : public _Accessor
+{
+  static_assert(::cuda::std::is_pointer_v<typename _Accessor::data_handle_type>, "Accessor must be pointer based");
+
+  using __data_handle_type = typename _Accessor::data_handle_type;
+  using __element_type     = ::cuda::std::remove_pointer_t<__data_handle_type>;
+
+  static constexpr bool __is_access_noexcept =
+    noexcept(::cuda::std::declval<_Accessor>().access(::cuda::std::declval<__data_handle_type>(), 0));
+
+  static constexpr bool __is_offset_noexcept =
+    noexcept(::cuda::std::declval<_Accessor>().offset(::cuda::std::declval<__data_handle_type>(), 0));
+
+public:
+  using offset_policy    = __shared_memory_accessor<typename _Accessor::offset_policy>;
+  using data_handle_type = __element_type*;
+  using reference        = typename _Accessor::reference;
+  using element_type     = typename _Accessor::element_type;
+
+  _CCCL_TEMPLATE(class _Accessor2 = _Accessor)
+  _CCCL_REQUIRES(::cuda::std::is_default_constructible_v<_Accessor2>)
+  _CCCL_API constexpr __shared_memory_accessor() noexcept(::cuda::std::is_nothrow_default_constructible_v<_Accessor2>)
+      : _Accessor{}
+  {
+    _CCCL_VERIFY_DEVICE_ONLY_USAGE();
+  }
+
+  _CCCL_API constexpr __shared_memory_accessor(const _Accessor& __acc) noexcept(
+    ::cuda::std::is_nothrow_copy_constructible_v<_Accessor>)
+      : _Accessor{__acc}
+  {
+    _CCCL_VERIFY_DEVICE_ONLY_USAGE();
+  }
+
+  _CCCL_API constexpr __shared_memory_accessor(_Accessor&& __acc) noexcept(
+    ::cuda::std::is_nothrow_move_constructible_v<_Accessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {
+    _CCCL_VERIFY_DEVICE_ONLY_USAGE();
+  }
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    ::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr __shared_memory_accessor(const __shared_memory_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {
+    _CCCL_VERIFY_DEVICE_ONLY_USAGE();
+  }
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, const _OtherAccessor&> _CCCL_AND(
+    !::cuda::std::is_convertible_v<const _OtherAccessor&, _Accessor>))
+  _CCCL_API constexpr explicit __shared_memory_accessor(const __shared_memory_accessor<_OtherAccessor>& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, const _OtherAccessor&>)
+      : _Accessor{__acc}
+  {
+    _CCCL_VERIFY_DEVICE_ONLY_USAGE();
+  }
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, _OtherAccessor> _CCCL_AND(
+    ::cuda::std::is_convertible_v<_OtherAccessor, _Accessor>))
+  _CCCL_API constexpr __shared_memory_accessor(__shared_memory_accessor<_OtherAccessor>&& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, _OtherAccessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {
+    _CCCL_VERIFY_DEVICE_ONLY_USAGE();
+  }
+
+  _CCCL_TEMPLATE(typename _OtherAccessor)
+  _CCCL_REQUIRES(::cuda::std::is_constructible_v<_Accessor, _OtherAccessor> _CCCL_AND(
+    !::cuda::std::is_convertible_v<_OtherAccessor, _Accessor>))
+  _CCCL_API constexpr explicit __shared_memory_accessor(__shared_memory_accessor<_OtherAccessor>&& __acc) noexcept(
+    ::cuda::std::is_nothrow_constructible_v<_Accessor, _OtherAccessor>)
+      : _Accessor{::cuda::std::move(__acc)}
+  {
+    _CCCL_VERIFY_DEVICE_ONLY_USAGE();
+  }
+
+  _CCCL_API reference access(__element_type* __p, ::cuda::std::size_t __i) const noexcept(__is_access_noexcept)
+  {
+#if !defined(_CCCL_DISABLE_MDSPAN_ACCESSOR_DETECT_INVALIDITY)
+    NV_IF_TARGET(
+      NV_IS_DEVICE,
+      (bool __is_shared_mem = ::__isShared(__p); //
+       _CCCL_ASSERT(__is_shared_mem, "__p is not a shared memory pointer");
+       _CCCL_ASSERT(__i <= ::cuda::__max_smem_allocation_bytes() / sizeof(__element_type),
+                    "__i exceeds the maximum shared memory allocation size");
+       _CCCL_ASSUME(__is_shared_mem);))
+    _CCCL_VERIFY_DEVICE_ONLY_USAGE();
+#endif // !defined(_CCCL_DISABLE_MDSPAN_ACCESSOR_DETECT_INVALIDITY)
+    return _Accessor::access(__p, __i);
+  }
+
+  _CCCL_API data_handle_type offset(__element_type* __p, ::cuda::std::size_t __i) const noexcept(__is_offset_noexcept)
+  {
+#if !defined(_CCCL_DISABLE_MDSPAN_ACCESSOR_DETECT_INVALIDITY)
+    NV_IF_TARGET(
+      NV_IS_DEVICE,
+      (bool __is_shared_mem = ::__isShared(__p); //
+       _CCCL_ASSERT(__is_shared_mem, "__p is not a shared memory pointer");
+       _CCCL_ASSERT(__i <= ::cuda::__max_smem_allocation_bytes() / sizeof(__element_type),
+                    "__i exceeds the maximum shared memory allocation size");
+       _CCCL_ASSUME(__is_shared_mem);))
+#endif // !defined(_CCCL_DISABLE_MDSPAN_ACCESSOR_DETECT_INVALIDITY)
+    _CCCL_VERIFY_DEVICE_ONLY_USAGE();
+    return _Accessor::offset(__p, __i);
+  }
+
+#if !defined(_CCCL_DISABLE_MDSPAN_ACCESSOR_DETECT_INVALIDITY)
+  [[nodiscard]] _CCCL_API static bool __detectably_invalid(
+    [[maybe_unused]] data_handle_type __p, [[maybe_unused]] ::cuda::std::size_t __size_bytes) noexcept
+  {
+    [[maybe_unused]] bool __is_valid = true;
+    NV_IF_TARGET(NV_IS_DEVICE,
+                 (bool __is_shared_mem     = ::cuda::device::is_address_from(__p, device::address_space::shared);
+                  bool __exceeds_smem_size = __size_bytes > ::cuda::__max_smem_allocation_bytes();
+                  __is_valid               = __is_shared_mem && !__exceeds_smem_size;))
+    _CCCL_ASSERT(__is_valid, "shared_memory_accessor (mdspan): data handle doesn't point to a valid shared memory");
+    _CCCL_VERIFY_DEVICE_ONLY_USAGE();
+    return !__is_valid;
+  }
+#endif // !defined(_CCCL_DISABLE_MDSPAN_ACCESSOR_DETECT_INVALIDITY)
+};
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MDSPAN_SHARED_MEMORY_ACCESSOR_H

cccl_upstream/libcudacxx/include/cuda/__mdspan/shared_memory_mdspan.h

@@ -0,0 +1,131 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MDSPAN_SHARED_MEMORY_MDSPAN_H
+#define _CUDA___MDSPAN_SHARED_MEMORY_MDSPAN_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__mdspan/shared_memory_accessor.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__fwd/array.h>
+#include <cuda/std/__fwd/span.h>
+#include <cuda/std/__mdspan/extents.h>
+#include <cuda/std/__mdspan/mdspan.h>
+#include <cuda/std/__type_traits/extent.h>
+#include <cuda/std/__type_traits/is_array.h>
+#include <cuda/std/__type_traits/is_convertible.h>
+#include <cuda/std/__type_traits/is_pointer.h>
+#include <cuda/std/__type_traits/rank.h>
+#include <cuda/std/__type_traits/remove_all_extents.h>
+#include <cuda/std/__type_traits/remove_pointer.h>
+#include <cuda/std/__type_traits/remove_reference.h>
+#include <cuda/std/__utility/delegate_constructors.h>
+#include <cuda/std/__utility/swap.h>
+#include <cuda/std/span> // __maybe_static_ext
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _ElementType,
+          typename _Extents,
+          typename _LayoutPolicy   = ::cuda::std::layout_right,
+          typename _AccessorPolicy = ::cuda::std::default_accessor<_ElementType>>
+class shared_memory_mdspan
+    : public ::cuda::std::mdspan<_ElementType, _Extents, _LayoutPolicy, shared_memory_accessor<_AccessorPolicy>>
+{
+public:
+  _CCCL_DELEGATE_CONSTRUCTORS(
+    shared_memory_mdspan,
+    ::cuda::std::mdspan,
+    _ElementType,
+    _Extents,
+    _LayoutPolicy,
+    shared_memory_accessor<_AccessorPolicy>);
+
+  _CCCL_API friend constexpr void swap(shared_memory_mdspan& __x, shared_memory_mdspan& __y) noexcept
+  {
+    swap(static_cast<__base&>(__x), static_cast<__base&>(__y));
+  }
+};
+
+_CCCL_TEMPLATE(typename _ElementType, typename... _OtherIndexTypes)
+_CCCL_REQUIRES((sizeof...(_OtherIndexTypes) > 0)
+                 _CCCL_AND(::cuda::std::is_convertible_v<_OtherIndexTypes, ::cuda::std::size_t>&&... && true))
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES explicit shared_memory_mdspan(_ElementType*, _OtherIndexTypes...)
+  -> shared_memory_mdspan<
+    _ElementType,
+    ::cuda::std::extents<::cuda::std::size_t, ::cuda::std::__maybe_static_ext<_OtherIndexTypes>...>>;
+
+_CCCL_TEMPLATE(typename _Pointer)
+_CCCL_REQUIRES(::cuda::std::is_pointer_v<::cuda::std::remove_reference_t<_Pointer>>)
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES shared_memory_mdspan(_Pointer&&)
+  -> shared_memory_mdspan<::cuda::std::remove_pointer_t<::cuda::std::remove_reference_t<_Pointer>>,
+                          ::cuda::std::extents<::cuda::std::size_t>>;
+
+_CCCL_TEMPLATE(typename _CArray)
+_CCCL_REQUIRES(::cuda::std::is_array_v<_CArray> _CCCL_AND(::cuda::std::rank_v<_CArray> == 1))
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES shared_memory_mdspan(_CArray&)
+  -> shared_memory_mdspan<::cuda::std::remove_all_extents_t<_CArray>,
+                          ::cuda::std::extents<::cuda::std::size_t, ::cuda::std::extent_v<_CArray, 0>>>;
+
+template <typename _ElementType, typename _OtherIndexType, ::cuda::std::size_t _Size>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES shared_memory_mdspan(_ElementType*, const ::cuda::std::array<_OtherIndexType, _Size>&)
+  -> shared_memory_mdspan<_ElementType, ::cuda::std::dextents<::cuda::std::size_t, _Size>>;
+
+template <typename _ElementType, typename _OtherIndexType, ::cuda::std::size_t _Size>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES shared_memory_mdspan(_ElementType*, ::cuda::std::span<_OtherIndexType, _Size>)
+  -> shared_memory_mdspan<_ElementType, ::cuda::std::dextents<::cuda::std::size_t, _Size>>;
+
+// This one is necessary because all the constructors take `data_handle_type`s, not
+// `_ElementType*`s, and `data_handle_type` is taken from `accessor_type::data_handle_type`, which
+// seems to throw off automatic deduction guides.
+template <typename _ElementType, typename _OtherIndexType, ::cuda::std::size_t... _ExtentsPack>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES
+shared_memory_mdspan(_ElementType*, const ::cuda::std::extents<_OtherIndexType, _ExtentsPack...>&)
+  -> shared_memory_mdspan<_ElementType, ::cuda::std::extents<_OtherIndexType, _ExtentsPack...>>;
+
+template <typename _ElementType, typename _MappingType>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES shared_memory_mdspan(_ElementType*, const _MappingType&)
+  -> shared_memory_mdspan<_ElementType, typename _MappingType::extents_type, typename _MappingType::layout_type>;
+
+template <typename _MappingType, typename _AccessorType>
+_CCCL_DEDUCTION_GUIDE_ATTRIBUTES
+shared_memory_mdspan(const typename _AccessorType::data_handle_type, const _MappingType&, const _AccessorType&)
+  -> shared_memory_mdspan<typename _AccessorType::element_type,
+                          typename _MappingType::extents_type,
+                          typename _MappingType::layout_type,
+                          _AccessorType>;
+
+/***********************************************************************************************************************
+ * Accessibility Traits
+ **********************************************************************************************************************/
+
+template <typename>
+inline constexpr bool is_shared_memory_mdspan_v = false;
+
+template <typename _Tp, typename _Ep, typename _Lp, typename _Ap>
+inline constexpr bool is_shared_memory_mdspan_v<shared_memory_mdspan<_Tp, _Ep, _Lp, _Ap>> = true;
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MDSPAN_SHARED_MEMORY_MDSPAN_H

cccl_upstream/libcudacxx/include/cuda/__mdspan/strides.h

@@ -0,0 +1,259 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MDSPAN_STRIDES_H
+#define _CUDA___MDSPAN_STRIDES_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__fwd/mdspan.h>
+#include <cuda/std/__concepts/concept_macros.h>
+#include <cuda/std/__cstddef/types.h>
+#include <cuda/std/__mdspan/extents.h>
+#include <cuda/std/__type_traits/is_convertible.h>
+#include <cuda/std/__type_traits/is_nothrow_constructible.h>
+#include <cuda/std/__type_traits/is_signed_integer.h>
+#include <cuda/std/__type_traits/make_unsigned.h>
+#include <cuda/std/__utility/cmp.h>
+#include <cuda/std/array>
+#include <cuda/std/span>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+// ------------------------------------------------------------------
+// ------------ strides ---------------------------------------------
+// ------------------------------------------------------------------
+
+//! @brief Class to describe the strides of a multi-dimensional array layout.
+//!
+//! Similar to extents, but for strides. Supports both static (compile-time known)
+//! and dynamic (runtime) stride values. Uses dynamic_stride as the tag for dynamic values.
+//!
+//! @tparam _OffsetType The signed integer type for stride values (supports negative strides)
+//! @tparam _Strides... The stride values, where dynamic_stride indicates a runtime value
+template <class _OffsetType, ::cuda::std::ptrdiff_t... _Strides>
+class strides
+    : private ::cuda::std::__mdspan_detail::
+        __maybe_static_array<_OffsetType, ::cuda::std::ptrdiff_t, dynamic_stride, _Strides...>
+{
+public:
+  using offset_type = _OffsetType;
+  using size_type   = ::cuda::std::make_unsigned_t<offset_type>;
+  using rank_type   = ::cuda::std::size_t;
+
+private:
+  static_assert(::cuda::std::__cccl_is_signed_integer_v<_OffsetType>,
+                "strides::offset_type must be a signed integer type");
+
+  template <class... _From>
+  [[nodiscard]] _CCCL_API static constexpr bool __is_representable_as(_From... __values) noexcept
+  {
+    return (
+      (::cuda::std::in_range<offset_type>(__values) || static_cast<::cuda::std::ptrdiff_t>(__values) == dynamic_stride)
+      && ...);
+  }
+
+  static_assert(__is_representable_as(_Strides...), "_Strides must be representable as offset_type");
+
+  static constexpr rank_type __rank_ = sizeof...(_Strides);
+  static constexpr rank_type __rank_dynamic_ =
+    ::cuda::std::__mdspan_detail::__count_dynamic_v<::cuda::std::ptrdiff_t, dynamic_stride, _Strides...>;
+
+  template <class _OtherIndexType>
+  static constexpr bool __is_convertible_to_index_type =
+    ::cuda::std::is_convertible_v<const _OtherIndexType&, offset_type>
+    && ::cuda::std::is_nothrow_constructible_v<offset_type, const _OtherIndexType&>;
+
+  using _Values =
+    ::cuda::std::__mdspan_detail::__maybe_static_array<_OffsetType, ::cuda::std::ptrdiff_t, dynamic_stride, _Strides...>;
+
+public:
+  [[nodiscard]] _CCCL_API static constexpr rank_type rank() noexcept
+  {
+    return __rank_;
+  }
+
+  [[nodiscard]] _CCCL_API static constexpr rank_type rank_dynamic() noexcept
+  {
+    return __rank_dynamic_;
+  }
+
+  [[nodiscard]] _CCCL_API constexpr offset_type stride(rank_type __r) const noexcept
+  {
+    return this->__value(__r);
+  }
+
+  [[nodiscard]] _CCCL_API static constexpr ::cuda::std::ptrdiff_t static_stride(rank_type __r) noexcept
+  {
+    return _Values::__static_value(__r);
+  }
+
+  _CCCL_HIDE_FROM_ABI constexpr strides() noexcept = default;
+
+  // Construction from just dynamic or all values
+  _CCCL_TEMPLATE(class... _OtherIndexTypes)
+  _CCCL_REQUIRES((sizeof...(_OtherIndexTypes) == __rank_ || sizeof...(_OtherIndexTypes) == __rank_dynamic_)
+                   _CCCL_AND(__is_convertible_to_index_type<_OtherIndexTypes>&&...))
+  _CCCL_API constexpr explicit strides(_OtherIndexTypes... __dynvals) noexcept
+      : _Values(static_cast<offset_type>(__dynvals)...)
+  {
+    _CCCL_ASSERT(__is_representable_as(__dynvals...), "strides ctor: arguments must be representable as offset_type");
+  }
+
+  _CCCL_TEMPLATE(class _OtherIndexType, ::cuda::std::size_t _Size)
+  _CCCL_REQUIRES((_Size == __rank_dynamic_) _CCCL_AND __is_convertible_to_index_type<_OtherIndexType>)
+  _CCCL_API constexpr strides(::cuda::std::span<_OtherIndexType, _Size> __strs) noexcept
+      : _Values(__strs)
+  {
+    for ([[maybe_unused]] const auto& __value : __strs)
+    {
+      _CCCL_ASSERT(__is_representable_as(__value), "strides ctor: arguments must be representable as offset_type");
+    }
+  }
+
+  _CCCL_TEMPLATE(class _OtherIndexType, ::cuda::std::size_t _Size)
+  _CCCL_REQUIRES((_Size != __rank_dynamic_) _CCCL_AND(_Size == __rank_)
+                   _CCCL_AND __is_convertible_to_index_type<_OtherIndexType>)
+  _CCCL_API explicit constexpr strides(::cuda::std::span<_OtherIndexType, _Size> __strs) noexcept
+      : _Values(__strs)
+  {
+    for ([[maybe_unused]] const auto& __value : __strs)
+    {
+      _CCCL_ASSERT(__is_representable_as(__value), "strides ctor: arguments must be representable as offset_type");
+    }
+  }
+
+  _CCCL_TEMPLATE(class _OtherIndexType, ::cuda::std::size_t _Size)
+  _CCCL_REQUIRES((_Size == __rank_dynamic_) _CCCL_AND __is_convertible_to_index_type<_OtherIndexType>)
+  _CCCL_API constexpr strides(const ::cuda::std::array<_OtherIndexType, _Size>& __strs) noexcept
+      : strides(::cuda::std::span<const _OtherIndexType, _Size>(__strs))
+  {}
+
+  _CCCL_TEMPLATE(class _OtherIndexType, ::cuda::std::size_t _Size)
+  _CCCL_REQUIRES((_Size == __rank_) _CCCL_AND(_Size != __rank_dynamic_)
+                   _CCCL_AND __is_convertible_to_index_type<_OtherIndexType>)
+  _CCCL_API explicit constexpr strides(const ::cuda::std::array<_OtherIndexType, _Size>& __strs) noexcept
+      : strides(::cuda::std::span<const _OtherIndexType, _Size>(__strs))
+  {}
+
+private:
+  // Helper to construct from other strides
+  template <::cuda::std::size_t _DynCount, ::cuda::std::size_t _Idx, class _OtherStrides, class... _DynamicValues>
+  [[nodiscard]] _CCCL_API constexpr _Values __construct_vals_from_strides(
+    [[maybe_unused]] const _OtherStrides& __strs, _DynamicValues... __dynamic_values) noexcept
+  {
+    if constexpr (_Idx == __rank_)
+    {
+      static_assert(_DynCount == __rank_dynamic_, "Constructor of invalid strides passed to strides::strides");
+      return _Values{static_cast<offset_type>(__dynamic_values)...};
+    }
+    else if constexpr (static_stride(_Idx) == dynamic_stride)
+    {
+      return __construct_vals_from_strides<_DynCount + 1, _Idx + 1>(__strs, __dynamic_values..., __strs.stride(_Idx));
+    }
+    else
+    {
+      return __construct_vals_from_strides<_DynCount, _Idx + 1>(__strs, __dynamic_values...);
+    }
+  }
+
+  struct __strides_delegate_tag
+  {};
+
+  template <class _OtherIndexType, ::cuda::std::ptrdiff_t... _OtherStrides>
+  _CCCL_API constexpr strides(__strides_delegate_tag, const strides<_OtherIndexType, _OtherStrides...>& __other) noexcept
+      : _Values(__construct_vals_from_strides<0, 0>(__other))
+  {
+    if constexpr (__rank_ != 0)
+    {
+      for (::cuda::std::size_t __r = 0; __r != __rank_; __r++)
+      {
+        _CCCL_ASSERT(__is_representable_as(__other.stride(__r)), "strides construction: stride is out of range");
+        _CCCL_ASSERT(_Values::__static_value(__r) == dynamic_stride
+                       || ::cuda::std::cmp_equal(__other.stride(__r), _Values::__static_value(__r)),
+                     "strides construction: mismatch of provided arguments with static strides.");
+      }
+    }
+  }
+
+  // Converting constructor from other strides specializations
+  template <class _OtherIndexType, ::cuda::std::ptrdiff_t... _OtherStrides>
+  static constexpr bool __is_explicit_conversion =
+    (((_Strides != dynamic_stride) && (_OtherStrides == dynamic_stride)) || ...);
+
+  template <::cuda::std::ptrdiff_t... _OtherStrides>
+  static constexpr bool __is_matching_strides =
+    ((_OtherStrides == dynamic_stride || _Strides == dynamic_stride || _OtherStrides == _Strides) && ...);
+
+public:
+  _CCCL_TEMPLATE(class _OtherIndexType, ::cuda::std::ptrdiff_t... _OtherStrides)
+  _CCCL_REQUIRES((sizeof...(_OtherStrides) == sizeof...(_Strides)) _CCCL_AND __is_matching_strides<_OtherStrides...>
+                   _CCCL_AND(!__is_explicit_conversion<_OtherIndexType, _OtherStrides...>))
+  _CCCL_API constexpr strides(const strides<_OtherIndexType, _OtherStrides...>& __other) noexcept
+      : strides(__strides_delegate_tag{}, __other)
+  {}
+
+  _CCCL_TEMPLATE(class _OtherIndexType, ::cuda::std::ptrdiff_t... _OtherStrides)
+  _CCCL_REQUIRES((sizeof...(_OtherStrides) == sizeof...(_Strides))
+                   _CCCL_AND __is_matching_strides<_OtherStrides...> _CCCL_AND
+                     __is_explicit_conversion<_OtherIndexType, _OtherStrides...>)
+  _CCCL_API explicit constexpr strides(const strides<_OtherIndexType, _OtherStrides...>& __other) noexcept
+      : strides(__strides_delegate_tag{}, __other)
+  {}
+
+  // Comparison operator
+  template <class _OtherIndexType, ::cuda::std::ptrdiff_t... _OtherStrides>
+  [[nodiscard]] _CCCL_API friend constexpr bool
+  operator==(const strides& __lhs, const strides<_OtherIndexType, _OtherStrides...>& __rhs) noexcept
+  {
+    if constexpr (__rank_ != sizeof...(_OtherStrides))
+    {
+      return false;
+    }
+    else
+    {
+      bool __result = true;
+      for (rank_type __r = 0; __r != __rank_; __r++)
+      {
+        if (::cuda::std::cmp_not_equal(__lhs.stride(__r), __rhs.stride(__r)))
+        {
+          __result = false;
+          break;
+        }
+      }
+      return __result;
+    }
+  }
+
+#if _CCCL_STD_VER <= 2017
+  template <class _OtherIndexType, ::cuda::std::ptrdiff_t... _OtherStrides>
+  [[nodiscard]] _CCCL_API friend constexpr bool
+  operator!=(const strides& __lhs, const strides<_OtherIndexType, _OtherStrides...>& __rhs) noexcept
+  {
+    return !(__lhs == __rhs);
+  }
+#endif // _CCCL_STD_VER <= 2017
+};
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MDSPAN_STRIDES_H

cccl_upstream/libcudacxx/include/cuda/__mdspan/traits.h

@@ -0,0 +1,53 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the libcu++ Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___MDSPAN_TRAITS_H
+#define _CUDA___MDSPAN_TRAITS_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__fwd/mdspan.h>
+#include <cuda/std/__fwd/mdspan.h>
+#include <cuda/std/__type_traits/is_same.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _LayoutPolicy>
+inline constexpr bool __is_layout_right_v = ::cuda::std::is_same_v<_LayoutPolicy, ::cuda::std::layout_right>;
+
+template <typename _LayoutPolicy>
+inline constexpr bool __is_layout_left_v = ::cuda::std::is_same_v<_LayoutPolicy, ::cuda::std::layout_left>;
+
+template <typename _LayoutPolicy>
+inline constexpr bool __is_layout_stride_v = ::cuda::std::is_same_v<_LayoutPolicy, ::cuda::std::layout_stride>;
+
+template <typename _LayoutPolicy>
+inline constexpr bool __is_layout_stride_relaxed_v =
+  ::cuda::std::is_same_v<_LayoutPolicy, ::cuda::layout_stride_relaxed>;
+
+template <typename _LayoutPolicy>
+inline constexpr bool __is_cuda_mdspan_layout_v =
+  __is_layout_right_v<_LayoutPolicy> || __is_layout_left_v<_LayoutPolicy> || __is_layout_stride_v<_LayoutPolicy>
+  || __is_layout_stride_relaxed_v<_LayoutPolicy>;
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___MDSPAN_TRAITS_H