feat(cccl): integrate missing CCCL directories — python/, ci/, .agent/, docs/, test/

Sparse-checkout from NVIDIA/cccl main branch to complete cccl_upstream:

Added:
- python/cuda_cccl/ (226 files) — Python bindings for device-level algorithms
  Critical for muh toolchain: cuda.compute.reduce_into, scan, radix_sort, etc.
  Includes 204 .py files with full test coverage for all 27 algorithms
- ci/ (163 files) — Build/test infrastructure
  build_cub.sh, test_cub.sh, build_and_test_targets.sh, matrix.yaml
  Directly maps to our [INFRA-CI] and [INFRA-BUILD] items
- .agent/skills/ (7 files) — NVIDIA's own agent skills for CCCL
  cccl-style/SKILL.md, cccl-test/SKILL.md, sass-diff/SKILL.md
- docs/ (491 files) — Official CCCL documentation
  CI references, CMake guides, Python compute docs, libcudacxx PTX docs
- test/ (12 files) — Top-level integration tests (cuda_smoke, stdpar)
- Root configs: .clang-format, .clang-tidy, CONTRIBUTING.md, pyproject.toml
- CLAUDE.md symlink → AGENTS.md (NVIDIA's standard)

cccl_upstream now mirrors full NVIDIA/cccl structure:
  Before: 42M (cub + thrust + libcudacxx + cudax + c + examples + benchmarks)
  After:  53M (+python +ci +docs +.agent +test +configs)

This completes the CCCL base needed for:
- [muh-bench] items: ci/util/build_and_test_targets.sh for targeted builds
- [CCCL-verify] items: python/cuda_cccl/tests/ as reference implementations
- [CCCL-test] items: ci/test_cub.sh, ci/test_thrust.sh
- Agent workflow: .agent/skills/ for consistent style and test patterns

cccl_upstream/docs/libcudacxx/Doxyfile

@@ -0,0 +1,182 @@
+# Doxyfile for libcudacxx
+
+PROJECT_NAME           = libcudacxx
+OUTPUT_DIRECTORY       = ../_build/doxygen/libcudacxx
+CREATE_SUBDIRS         = NO
+GENERATE_HTML          = NO
+GENERATE_LATEX         = NO
+GENERATE_XML           = YES
+XML_OUTPUT             = xml
+XML_PROGRAMLISTING     = YES
+
+INPUT                  = ../../libcudacxx/include/cuda/__algorithm \
+                         ../../libcudacxx/include/cuda/__container \
+                         ../../libcudacxx/include/cuda/__device \
+                         ../../libcudacxx/include/cuda/__event \
+                         ../../libcudacxx/include/cuda/__hierarchy \
+                         ../../libcudacxx/include/cuda/__iterator \
+                         ../../libcudacxx/include/cuda/__launch \
+                         ../../libcudacxx/include/cuda/__memory_pool \
+                         ../../libcudacxx/include/cuda/__memory_resource \
+                         ../../libcudacxx/include/cuda/__stream \
+                         ../../libcudacxx/include/nv \
+                         ../../libcudacxx/include/cuda/std/__cccl/architecture.h \
+                         ../../libcudacxx/include/cuda/std/__cccl/os.h
+
+RECURSIVE              = YES
+EXCLUDE_PATTERNS       = */detail/* */test/* */tests/* */examples/* */__detail/* */std/__cccl/compiler.h
+EXCLUDE_SYMBOLS        = *detail* *__* _LIBCUDACXX_*
+
+FILE_PATTERNS          = *.h *.hpp *.cuh
+EXTENSION_MAPPING      = cuh=C++ cu=C++
+
+# Documentation extraction settings
+EXTRACT_ALL            = YES
+EXTRACT_PRIVATE        = NO
+EXTRACT_STATIC         = YES
+EXTRACT_LOCAL_CLASSES  = YES
+HIDE_UNDOC_MEMBERS     = NO
+HIDE_UNDOC_CLASSES     = NO
+SHOW_INCLUDE_FILES     = YES
+INLINE_INHERITED_MEMB  = YES
+FULL_PATH_NAMES        = YES
+STRIP_FROM_PATH        = ../../libcudacxx
+SHORT_NAMES            = NO
+
+# Parsing settings
+JAVADOC_AUTOBRIEF      = YES
+QT_AUTOBRIEF           = NO
+MULTILINE_CPP_IS_BRIEF = NO
+INHERIT_DOCS           = YES
+SEPARATE_MEMBER_PAGES  = NO
+TAB_SIZE               = 4
+BUILTIN_STL_SUPPORT    = YES
+
+# Preprocessing
+ENABLE_PREPROCESSING   = YES
+MACRO_EXPANSION        = YES
+EXPAND_ONLY_PREDEF     = NO
+SEARCH_INCLUDES        = YES
+SKIP_FUNCTION_MACROS   = YES
+
+# Predefined macros for libcudacxx
+PREDEFINED = \
+    "__device__=" \
+    "__host__=" \
+    "__global__=" \
+    "__forceinline__=" \
+    "__declspec(x)=" \
+    "__align__(x)=" \
+    "_CCCL_DOXYGEN_INVOKED" \
+    "_CCCL_HOST_DEVICE=" \
+    "_CCCL_DEVICE=" \
+    "_CCCL_HOST=" \
+    "_CCCL_FORCEINLINE=" \
+    "_CCCL_CONSTEXPR_CXX14=constexpr" \
+    "_CCCL_CONSTEXPR_CXX17=constexpr" \
+    "_CCCL_CONSTEXPR_CXX20=constexpr" \
+    "_CCCL_CONSTEXPR_CXX23=constexpr" \
+    "_LIBCUDACXX_HIDE_FROM_ABI=" \
+    "_LIBCUDACXX_INLINE_VISIBILITY=" \
+    "_LIBCUDACXX_TEMPLATE_VIS=" \
+    "__cccl_lib_mdspan" \
+    "_CCCL_AND=&&" \
+    "_CCCL_API=inline" \
+    "_CCCL_HOST_DEVICE_API=inline" \
+    "_CCCL_BEGIN_NAMESPACE_ARCH_DEPENDENT="\
+    "_CCCL_BEGIN_NAMESPACE_CUDA=namespace cuda {" \
+    "_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE=namespace cuda::device {" \
+    "_CCCL_BEGIN_NAMESPACE_CUDA_DEVICE_EXPERIMENTAL=namespace cuda::device::experimental {" \
+    "_CCCL_BEGIN_NAMESPACE_CUDA_DRIVER=namespace cuda::__driver {" \
+    "_CCCL_BEGIN_NAMESPACE_CUDA_EXECUTION=namespace cuda::execution {" \
+    "_CCCL_BEGIN_NAMESPACE_CUDA_MR=namespace cuda::mr {" \
+    "_CCCL_BEGIN_NAMESPACE_CUDA_PTX =namespace cuda::ptx {" \
+    "_CCCL_BEGIN_NAMESPACE_CUDA_STD=namespace cuda::std {" \
+    "_CCCL_BEGIN_NAMESPACE_CUDA_STD_EXECUTION=namespace cuda::std::execution {" \
+    "_CCCL_BEGIN_NAMESPACE_CUDA_STD_RANGES=namespace cuda::std::ranges {" \
+    "_CCCL_BEGIN_NAMESPACE_CUDA_STD_VIEWS=namespace cuda::std::ranges::views {" \
+    "_CCCL_BEGIN_NAMESPACE_STD=namespace std {" \
+    "_CCCL_CATCH=catch" \
+    "_CCCL_CATCH_ALL=catch (...)" \
+    "_CCCL_CATCH_FALLTHROUGH=" \
+    "_CCCL_CONCEPT=constexpr bool " \
+    "_CCCL_CONSTEXPR_FRIEND=friend " \
+    "_CCCL_CTK_AT_LEAST(x, y)=1" \
+    "_CCCL_CTK_BELOW(x, y)=0" \
+    "_CCCL_CUDACC_AT_LEAST(x, y)=1" \
+    "_CCCL_CUDACC_BELOW(x, y)=0" \
+    "_CCCL_DEVICE_API=inline" \
+    "_CCCL_DIAG_POP=" \
+    "_CCCL_DIAG_PUSH=" \
+    "_CCCL_DIAG_SUPPRESS_CLANG(x)=" \
+    "_CCCL_DIAG_SUPPRESS_GCC(x)=" \
+    "_CCCL_DIAG_SUPPRESS_MSVC(x)=" \
+    "_CCCL_DIAG_SUPPRESS_NVHPC(x)=" \
+    "_CCCL_END_NAMESPACE_ARCH_DEPENDENT="\
+    "_CCCL_END_NAMESPACE_CUDA=}" \
+    "_CCCL_END_NAMESPACE_CUDA_DEVICE=}" \
+    "_CCCL_END_NAMESPACE_CUDA_DEVICE_EXPERIMENTAL=}" \
+    "_CCCL_END_NAMESPACE_CUDA_DRIVER=}" \
+    "_CCCL_END_NAMESPACE_CUDA_EXECUTION=}" \
+    "_CCCL_END_NAMESPACE_CUDA_MR=}" \
+    "_CCCL_END_NAMESPACE_CUDA_PTX=}" \
+    "_CCCL_END_NAMESPACE_CUDA_STD=}" \
+    "_CCCL_END_NAMESPACE_CUDA_STD_EXECUTION=}" \
+    "_CCCL_END_NAMESPACE_CUDA_STD_RANGES=}" \
+    "_CCCL_END_NAMESPACE_CUDA_STD_VIEWS=}" \
+    "_CCCL_END_NAMESPACE_STD=}" \
+    "_CCCL_EXEC_CHECK_DISABLE=" \
+    "_CCCL_GLOBAL_CONSTANT=inline constexpr" \
+    "_CCCL_HAS_CTK()=1" \
+    "_CCCL_HIDE_FROM_ABI=" \
+    "_CCCL_HOST_API=inline" \
+    "_CCCL_PUBLIC_API=inline" \
+    "_CCCL_PUBLIC_DEVICE_API=inline" \
+    "_CCCL_PUBLIC_HOST_API=inline" \
+    "_CCCL_REQUIRES(x)= ::cuda::std::enable_if_t<x, int> = 0>" \
+    "_CCCL_STD_VER=2020" \
+    "_CCCL_SUPPRESS_DEPRECATED_POP=" \
+    "_CCCL_SUPPRESS_DEPRECATED_PUSH=" \
+    "_CCCL_TEMPLATE(x)=template<x, " \
+    "_CCCL_TRAILING_REQUIRES(x)=-> x requires " \
+    "_CCCL_NODEBUG_API=inline" \
+    "_CCCL_NODEBUG_DEVICE_API=inline" \
+    "_CCCL_NODEBUG_HOST_API=inline" \
+    "_CCCL_TRY=try" \
+    "_CCCL_TYPE_VISIBILITY_DEFAULT=" \
+    "_CCCL_TYPE_VISIBILITY_HIDDEN=" \
+    "_CCCL_VISIBILITY_DEFAULT=" \
+    "_CCCL_VISIBILITY_HIDDEN=" \
+    "_CCCL_LIFETIMEBOUND=" \
+    "_CUDAX_CONSTEXPR_FRIEND=friend" \
+    "_LIBCUDACXX_HAS_SPACESHIP_OPERATOR()=0" \
+    "CCCL_DEPRECATED=" \
+    "CCCL_DEPRECATED_BECAUSE(x)=" \
+    "CCCL_IGNORE_DEPRECATED_CPP_DIALECT" \
+    "CUB_DISABLE_NAMESPACE_MAGIC" \
+    "CUB_IGNORE_NAMESPACE_MAGIC_ERROR" \
+    "CUB_NAMESPACE_BEGIN=namespace cub {" \
+    "CUB_NAMESPACE_END=}" \
+    "CUB_RUNTIME_FUNCTION=" \
+    "THRUST_FWD(x)=x" \
+    "THRUST_NAMESPACE_BEGIN=namespace thrust {" \
+    "THRUST_NAMESPACE_END=}" \
+    "THRUST_PREVENT_MACRO_SUBSTITUTION" \
+    "_CCCL_HOSTED()=1"
+
+# IMPORTANT: Aliases for custom commands
+# The rst alias enables embedding reStructuredText in doxygen comments
+# Using the same format as repo_docs for compatibility
+ALIASES = "rst=\verbatim embed:rst:leading-asterisk"
+ALIASES += "endrst=\endverbatim"
+ALIASES += "rststar=\verbatim embed:rst:leading-asterisk"
+ALIASES += "inlinerst=\verbatim embed:rst:inline"
+
+# Quiet mode
+QUIET                  = YES
+WARNINGS               = YES
+WARN_AS_ERROR          = FAIL_ON_WARNINGS
+WARN_IF_UNDOCUMENTED   = NO
+WARN_IF_DOC_ERROR      = YES
+WARN_IF_INCOMPLETE_DOC = NO
+WARN_NO_PARAMDOC       = NO

cccl_upstream/docs/libcudacxx/extended_api.rst

@@ -0,0 +1,31 @@
+.. _libcudacxx-extended-api:
+
+Extended API
+============
+
+.. toctree::
+   :maxdepth: 2
+
+   extended_api/macros
+   extended_api/bit
+   extended_api/execution_model
+   extended_api/exceptions
+   extended_api/memory_model
+   extended_api/thread_groups
+   extended_api/synchronization_primitives
+   extended_api/asynchronous_operations
+   extended_api/memory_access_properties
+   extended_api/functional
+   extended_api/iterators
+   extended_api/type_traits
+   extended_api/vector_tuple_protocol
+   extended_api/numeric
+   extended_api/random
+   extended_api/memory
+   extended_api/memory_resource
+   extended_api/math
+   extended_api/mdspan
+   extended_api/tma
+   extended_api/warp
+   extended_api/utility
+   extended_api/work_stealing

cccl_upstream/docs/libcudacxx/extended_api/asynchronous_operations.rst

@@ -0,0 +1,35 @@
+.. _libcudacxx-extended-api-asynchronous-operations:
+
+Asynchronous Operations
+-----------------------
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   asynchronous_operations/memcpy_async_tx
+   asynchronous_operations/memcpy_async
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`cuda::memcpy_async <libcudacxx-extended-api-asynchronous-operations-memcpy-async>`
+     - Asynchronously copies one range to another
+     - libcu++ 1.1.0 / CCCL 2.0.0
+     - CUDA 11.0
+
+   * - :ref:`cuda::memcpy_async_tx <libcudacxx-extended-api-asynchronous-operations-memcpy-async-tx>`
+     - Asynchronously copies one range to another with manual transaction accounting
+     - libcu++ 1.2.0 / CCCL 2.0.0
+     - CUDA 11.1
+
+.. note::
+
+  **Asynchronous operations** like `memcpy_async <libcudacxx-extended-api-asynchronous-operations-memcpy-async>`
+  are non-blocking operations performed as-if by a new thread of execution.

cccl_upstream/docs/libcudacxx/extended_api/asynchronous_operations/memcpy_async.rst

@@ -0,0 +1,278 @@
+.. _libcudacxx-extended-api-asynchronous-operations-memcpy-async:
+
+``cuda::memcpy_async``
+======================
+
+Defined in header ``<cuda/barrier>``:
+
+.. code:: cuda
+
+   // (1)
+   template <typename Shape, cuda::thread_scope Scope, typename CompletionFunction>
+   __host__ __device__
+   void cuda::memcpy_async(void* destination, void const* source, Shape size,
+                           cuda::barrier<Scope, CompletionFunction>& barrier);
+
+   // (2)
+   template <typename Group,
+             typename Shape, cuda::thread_scope Scope, typename CompletionFunction>
+   __host__ __device__
+   void cuda::memcpy_async(Group const& group,
+                           void* destination, void const* source, Shape size,
+                           cuda::barrier<Scope, CompletionFunction>& barrier);
+
+Defined in header ``<cuda/pipeline>``:
+
+.. code:: cuda
+
+   // (3)
+   template <typename Shape, cuda::thread_scope Scope>
+   __host__ __device__
+   void cuda::memcpy_async(void* destination, void const* source, Shape size,
+                           cuda::pipeline<Scope>& pipeline);
+
+   // (4)
+   template <typename Group, typename Shape, cuda::thread_scope Scope>
+   __host__ __device__
+   void cuda::memcpy_async(Group const& group,
+                           void* destination, void const* source, Shape size,
+                           cuda::pipeline<Scope>& pipeline);
+
+Defined in header ``<cuda/annotated_ptr>``:
+
+.. code:: cuda
+
+   // (5)
+   template <typename Dst, typename Src, typename SrcProperty, typename Shape, typename Sync>
+   __host__ __device__
+   void memcpy_async(Dst* dst, cuda::annotated_ptr<Src, SrcProperty> src, Shape size, Sync& sync);
+
+   // (6)
+   template<typename Dst, typename DstProperty, typename Src, typename SrcProperty, typename Shape, typename Sync>
+   __host__ __device__
+   void memcpy_async(cuda::annotated_ptr<Dst, DstProperty> dst, cuda::annotated_ptr<Src, SrcProperty> src, Shape size, Sync& sync);
+
+   // (7)
+   template<typename Group, typename Dst, typename Src, typename SrcProperty, typename Shape, typename Sync>
+   __host__ __device__
+   void memcpy_async(Group const& group, Dst* dst, cuda::annotated_ptr<Src, SrcProperty> src, Shape size, Sync& sync);
+
+   // (8)
+   template<typename Group, typename Dst, typename DstProperty, typename Src, typename SrcProperty, typename Shape, typename Sync>
+   __host__ __device__
+   void memcpy_async(Group const& group, cuda::annotated_ptr<Dst, DstProperty> dst, cuda::annotated_ptr<Src, SrcProperty> src, Shape size, Sync& sync);
+
+``cuda::memcpy_async`` asynchronously copies ``size`` bytes from the
+memory location pointed to by ``source`` to the memory location pointed
+to by ``destination``. Both objects are reinterpreted as arrays of
+``unsigned char``.
+
+1. Non-group version. Binds the asynchronous copy completion to ``cuda::barrier`` and
+   issues the copy in the current thread.
+2. Group version. Binds the asynchronous copy completion to ``cuda::barrier`` and
+   cooperatively issues the copy across all threads in ``group``.
+3. Non-group version. Binds the asynchronous copy completion to ``cuda::pipeline`` and
+   issues the copy in the current thread.
+4. Group version. Binds the asynchronous copy completion to ``cuda::pipeline`` and
+   cooperatively issues the copy across all threads in ``group``.
+5. 5-8: convenience wrappers using ``cuda::annotated_ptr`` where
+   ``Sync`` is either ``cuda::barrier`` or ``cuda::pipeline``.
+
+Notes
+-----
+
+``cuda::memcpy_async`` have similar constraints to `std::memcpy <https://en.cppreference.com/w/cpp/string/byte/memcpy>`_,
+namely:
+
+   - If the objects overlap, the behavior is undefined.
+   - If either ``destination`` or ``source`` is an invalid or null pointer, the behavior is undefined
+     (even if ``count`` is zero).
+   - If the objects are `potentially-overlapping <https://en.cppreference.com/w/cpp/language/object#Subobjects>`_
+     the behavior is undefined.
+   - If the objects are not of `TriviallyCopyable <https://en.cppreference.com/w/cpp/named_req/TriviallyCopyable>`_
+     type the program is ill-formed, no diagnostic required.
+
+Additionally:
+
+   - If *Shape* is :ref:`cuda::aligned_size_t <libcudacxx-extended-api-memory-aligned-size>`, ``source``
+     and ``destination`` are both required to be aligned on ``cuda::aligned_size_t::align``, else the behavior is
+     undefined.
+   - If ``cuda::pipeline`` is in a *quitted state*
+     (see :ref:`cuda::pipeline::quit <libcudacxx-extended-api-synchronization-pipeline-pipeline-quit>`),
+     the behavior is undefined.
+   - For cooperative overloads (with a group parameter),
+     if the parameters are not the same across all threads in ``group``,
+     or not all threads represented by ``group`` call the overload, the behavior is undefined.
+   - The group of a cooperative overload can also represent a partition of the active threads calling the overload,
+     in which case a copy is cooperatively issued per partition of the active threads described by ``group``.
+     For example, if ``group`` is a ``cooperative_groups::thread_block_tile<32, ...>``
+     and the overload is called with 128 threads active, 4 copies will be issued, one cooperatively per warp.
+   - If a non-group overload is called with multiple threads active,
+     each thread issues its own copy and thus must have different arguments and the copies must not overlap.
+
+
+Template Parameters
+-------------------
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``Group``
+     - A type satisfying the [*Group*] concept.
+   * - ``Shape``
+     - Either `cuda::std::size_t <https://en.cppreference.com/w/c/types/size_t>`_
+       or :ref:`cuda::aligned_size_t <libcudacxx-extended-api-memory-aligned-size>`.
+
+Parameters
+----------
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``group``
+     - The group of threads.
+   * - ``destination``
+     - Pointer to the memory location to copy to.
+   * - ``source``
+     - Pointer to the memory location to copy from.
+   * - ``size``
+     - The number of bytes to copy.
+   * - ``barrier``
+     - The barrier object used to wait on the copy completion.
+   * - ``pipeline``
+     - The pipeline object used to wait on the copy completion.
+
+Related traits
+--------------
+
+.. code:: cuda
+
+   template <typename Group>
+   constexpr inline bool is_thread_block_group_v;
+
+This trait is ``true`` if ``Group`` represents the full CUDA thread block.
+For example, ``cooperative_groups::thread_block`` satisfies this trait.
+Users are encouraged to specialize this trait for their own groups.
+
+.. code:: cuda
+
+   template <typename Group>
+   constexpr inline bool is_warp_group_v = false;
+
+This trait is ``true`` if ``Group`` represents a full CUDA warp.
+For example, ``cooperative_groups::thread_block_tile<32, ...>`` satisfies this trait.
+Users are encouraged to specialize this trait for their own groups.
+
+Implementation notes
+--------------------
+
+On Hopper+ GPUs, the overloads taking a barrier may use the Tensor Memory Accelerator (TMA)
+via the ``cp.async.bulk`` instruction to perform the copy if:
+- the barrier resides in shared memory,
+- the data is aligned to 16 bytes,
+- the source is global memory,
+- the destination is shared memory.
+Additionally, the cooperative overload (taking a group) can generate more efficient code
+if the group satisfies the trait ``cuda::is_thread_block_group_v`` or ``cuda::is_warp_group_v``.
+In those cases, a uniform data path is generated for the bulk copy and thread peeling is avoided.
+
+On Ampere+ GPUs, the ``cp.async`` instruction may be used to perform the copy if:
+- the data is aligned to at least 4 bytes,
+- the source is global memory,
+- the destination is shared memory.
+
+
+Examples
+--------
+
+.. rubric:: Example: Using a system-wide barrier to copy within global memory
+
+.. code:: cuda
+
+   #include <cuda/barrier>
+
+   __global__ void example_kernel(char* dst, char* src) {
+     cuda::barrier<cuda::thread_scope_system> bar;
+     init(&bar, 1);
+
+     cuda::memcpy_async(dst,     src,      1, bar);
+     cuda::memcpy_async(dst + 1, src + 8,  1, bar);
+     cuda::memcpy_async(dst + 2, src + 16, 1, bar);
+     cuda::memcpy_async(dst + 3, src + 24, 1, bar);
+
+     bar.arrive_and_wait();
+   }
+
+`See it on Godbolt <https://godbolt.org/z/od6q9s8fq>`_
+
+.. rubric:: Example: 1D load of two buffers from global to shared memory with a barrier
+
+.. code:: cuda
+
+   #include <cuda/barrier>
+
+   __global__ void example_kernel(int* gmem1, double* gmem2) {
+     constexpr int tile_size = 1024;
+     __shared__ alignas(16)    int smem1[tile_size];
+     __shared__ alignas(16) double smem2[tile_size];
+
+     #pragma nv_diag_suppress static_var_with_dynamic_init
+     __shared__  cuda::barrier<cuda::thread_scope_block> bar;
+
+     // setup the barrier where each thread in the block arrives at
+     if (threadIdx.x == 0) {
+       init(&bar, blockDim.x);
+     }
+     __syncthreads();
+
+     // issue two copy operations
+     auto group = ...;
+     cuda::memcpy_async(group, smem1, gmem1, cuda::aligned_size_t<16>(tile_size * sizeof(int)   ), bar);
+     cuda::memcpy_async(group, smem2, gmem2, cuda::aligned_size_t<16>(tile_size * sizeof(double)), bar);
+
+     // arrive and wait for copy operations to complete
+     bar.arrive_and_wait();
+
+     // process data in smem ...
+   }
+
+There are multiple possibilities to initialize the ``group`` variable.
+One option is to use the cooperative groups API:
+
+.. code:: cuda
+
+    #include <cooperative_groups.h>
+    auto group = cooperative_groups::this_thread_block();
+
+Another option, especially if the dimensionality of the thread block is known, e.g. 1D,
+a custom group can be defined like:
+
+.. code:: cuda
+
+   struct this_thread_block_1D {
+     static constexpr cuda::thread_scope thread_scope = cuda::thread_scope_block;
+
+     __device__ void sync() const {
+       __syncthreads();
+     }
+
+     __device__ auto size() const {
+       return blockDim.x;
+     }
+
+     __device__ auto thread_rank() const {
+       return threadIdx.x;
+     }
+   };
+
+   template <>
+   inline constexpr bool cuda::is_thread_block_group_v<this_thread_block_1D> = true;
+
+Such a group will emit the least amount of code when used with ``cuda::memcpy_async``,
+since the ``thread_rank()`` is easily computed (because the block is 1D)
+and we declared the group as representing the whole thread block,
+which allows emit a uniform data path on Hopper+ GPUs in certain conditions.
+
+`See it on Godbolt <https://godbolt.org/z/aM9cbabcW>`__

cccl_upstream/docs/libcudacxx/extended_api/asynchronous_operations/memcpy_async_tx.rst

@@ -0,0 +1,93 @@
+.. _libcudacxx-extended-api-asynchronous-operations-memcpy-async-tx:
+
+``cuda::device::memcpy_async_tx``
+=================================
+
+Defined in header ``<cuda/barrier>``:
+
+.. code:: cuda
+
+   template <typename T, size_t Alignment>
+   inline __device__
+   void cuda::device::memcpy_async_tx(
+     T* dest,
+     const T* src,
+     cuda::aligned_size_t<Alignment> size,
+     cuda::barrier<cuda::thread_scope_block>& bar);
+
+Copies ``size`` bytes from global memory ``src`` to shared memory ``dest`` and decrements the transaction count of ``bar`` by ``size`` bytes.
+
+Preconditions
+-------------
+
+-  ``src``, ``dest`` are 16-byte aligned and ``size`` is a multiple of 16, i.e., ``Alignment >= 16``.
+-  ``dest`` points to a shared memory allocation that is at least ``size`` bytes wide.
+-  ``src`` points to a global memory allocation that is at least ``size`` bytes wide.
+-  ``bar`` is located in shared memory
+-  If either ``destination`` or ``source`` is an invalid or null pointer, the behavior is undefined (even if ``count`` is zero).
+
+Requires
+--------
+
+-  ``is_trivially_copyable_v<T>`` is true.
+
+Notes
+-----
+
+This function can only be used under CUDA Compute Capability 9.0 (Hopper) or higher.
+
+There is no feature flag to check if ``cuda::device::memcpy_async_tx`` is available.
+
+**Comparison to cuda::memcpy_async**: ``memcpy_async_tx`` supports a subset of the operations of ``memcpy_async``.
+It gives more control over the synchronization with a barrier than ``memcpy_async``.
+Currently, ``memcpy_async_tx`` has no synchronous fallback mechanism., i.e., it currently does not work on older hardware
+(pre-CUDA Compute Capability 9.0, i.e., Hopper).
+
+.. _libcudacxx-extended-api-asynchronous-operations-memcpy-async-tx-example:
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/barrier>
+   #include <cuda/std/utility> // cuda::std::move
+
+   #if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900
+   static_assert(false, "Insufficient CUDA Compute Capability: cuda::device::memcpy_async_tx is not available.");
+   #endif // __CUDA_MINIMUM_ARCH__
+
+   __device__ alignas(16) int gmem_x[2048];
+
+   __device__ inline bool elect_one() {
+     const unsigned int tid = threadIdx.x;
+     const unsigned int warp_id = tid / 32;
+     const unsigned int uniform_warp_id = __shfl_sync(0xFFFFFFFF, warp_id, 0); // broadcast from lane 0
+     return (uniform_warp_id == 0 && cuda::ptx::elect_sync(0xFFFFFFFF)); // elect a leader thread among warp 0
+   }
+
+   __global__ void example_kernel() {
+     alignas(16) __shared__ int smem_x[1024];
+     #pragma nv_diag_suppress static_var_with_dynamic_init
+     __shared__ cuda::barrier<cuda::thread_scope_block> bar;
+
+     // setup the mbarrier
+     if (threadIdx.x == 0) {
+       init(&bar, blockDim.x);
+     }
+     __syncthreads();
+
+     // issue the async copy from a single thread and wait for completion
+     const bool is_block_leader = elect_one();
+     const int tx_count = is_block_leader ? sizeof(smem_x) : 0;
+     if (is_block_leader) {
+       cuda::device::memcpy_async_tx(smem_x, gmem_x, cuda::aligned_size_t<16>(tx_count), bar);
+     }
+     auto token = cuda::device::barrier_arrive_tx(bar, 1, tx_count);
+     bar.wait(cuda::std::move(token));
+
+     // smem_x contains the contents of gmem_x[0], ..., gmem_x[1023]
+     smem_x[threadIdx.x] += 1;
+   }
+
+`See it on Godbolt <https://godbolt.org/z/M8zqnrz9b>`_

cccl_upstream/docs/libcudacxx/extended_api/bit.rst

@@ -0,0 +1,42 @@
+.. _libcudacxx-extended-api-bit:
+
+Bit
+===
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   bit/bitmask
+   bit/bit_reverse
+   bit/bitfield_insert
+   bit/bitfield_extract
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`bitmask <libcudacxx-extended-api-bit-bitmask>`
+     - Generate a bitmask
+     - CCCL 3.0.0
+     - CUDA 13.0
+
+   * - :ref:`bit_reverse <libcudacxx-extended-api-bit-bit_reverse>`
+     - Reverse the order of bits
+     - CCCL 3.0.0
+     - CUDA 13.0
+
+   * - :ref:`bitfield_insert <libcudacxx-extended-api-bit-bitfield_insert>`
+     - Insert a bitfield
+     - CCCL 3.0.0
+     - CUDA 13.0
+
+   * - :ref:`bitfield_extract <libcudacxx-extended-api-bit-bitfield_extract>`
+     - Extract a bitfield
+     - CCCL 3.0.0
+     - CUDA 13.0

cccl_upstream/docs/libcudacxx/extended_api/bit/bit_reverse.rst

@@ -0,0 +1,72 @@
+.. _libcudacxx-extended-api-bit-bit_reverse:
+
+``cuda::bit_reverse``
+=====================
+
+Defined in the ``<cuda/bit>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ constexpr
+   T bit_reverse(T value) noexcept;
+
+   } // namespace cuda
+
+The function reverses the order of bits in a value.
+
+**Parameters**
+
+- ``value``: Input value
+
+**Return value**
+
+- Value with reversed bits
+
+**Constraints**
+
+- ``T`` is an unsigned integer type.
+
+**Performance considerations**
+
+The function performs the following operations:
+
+- Device:
+
+  - ``uint8_t`` ``uint16_t``: ``PRMT``, ``BREV``
+  - ``uint32_t``: ``BREV``
+  - ``uint64_t``: ``BREV`` x2, ``MOV`` x2
+  - ``uint128_t``: ``BREV`` x4, ``MOV`` x4
+
+- Host: ``__builtin_bitreverse<N>`` with clang
+
+.. note::
+
+    Using the function could not be optimal when the input is a run-time value that the compiler can resolve at compile-time, e.g. an index of a loop with a fixed number of iterations.
+
+.. note::
+
+    GCC <= 8 uses a slow path with more instructions even in device code.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/bit>
+    #include <cuda/std/cassert>
+
+    __global__ void bit_reverse_kernel() {
+        assert(cuda::bit_reverse(0x0000FFFFu) == 0xFFFF0000u);
+        assert(cuda::bit_reverse(uint8_t{0b00001011}) == uint8_t{0b11010000});
+    }
+
+    int main() {
+        bit_reverse_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/nW6qe5fT4>`__

cccl_upstream/docs/libcudacxx/extended_api/bit/bitfield_extract.rst

@@ -0,0 +1,75 @@
+.. _libcudacxx-extended-api-bit-bitfield_extract:
+
+``cuda::bitfield_extract``
+==========================
+
+Defined in the ``<cuda/bit>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ constexpr
+   T bitfield_extract(T value, int start, int width) noexcept;
+
+   } // namespace cuda
+
+The function extracts a bitfield from a value and returns it in the lower bits.
+``bitfield_extract()`` computes ``(value >> start) & mask``, where ``mask`` is a sequence of bits of width ``width``.
+
+**Parameters**
+
+- ``value``: The value to apply the bitfield.
+- ``start``:  Initial position of the bitfield.
+- ``width``:  Width of the bitfield.
+
+**Return value**
+
+- ``(value >> start) & mask``, where ``mask`` is a bitmask of width ``width``.
+
+**Constraints**
+
+- ``T`` is an unsigned integer type.
+
+**Preconditions**
+
+- ``start >= 0 && start <= num_bits(T)``.
+- ``width >= 0 && width <= num_bits(T)``.
+- ``start + width <= num_bits(T)``.
+
+**Performance considerations**
+
+The function performs the following operations in CUDA for ``uint8_t``, ``uint16_t``, ``uint32_t``:
+
+- ``SM < 70``: ``BFE``.
+- ``SM >= 70``: ``BMSK``, bitwise operation x2.
+
+.. note::
+
+    Using the function could not be optimal when the input is a run-time value that the compiler can resolve at compile-time, e.g. an index of a loop with a fixed number of iterations.
+
+.. note::
+
+    GCC <= 8 uses a slow path with more instructions even in CUDA.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/bit>
+    #include <cuda/std/cassert>
+
+    __global__ void bitfield_extract_kernel() {
+        assert(cuda::bitfield_extract(~0u, 0, 4) == 0b1111);
+        assert(cuda::bitfield_extract(0b1011000u, 3, 4) == 0b1011);
+    }
+
+    int main() {
+        bitfield_extract_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/ofGnbsxW5>`__

cccl_upstream/docs/libcudacxx/extended_api/bit/bitfield_insert.rst

@@ -0,0 +1,76 @@
+.. _libcudacxx-extended-api-bit-bitfield_insert:
+
+``cuda::bitfield_insert``
+=========================
+
+Defined in the ``<cuda/bit>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ constexpr
+   T bitfield_insert(T dest, T source, int start, int width) noexcept;
+
+   } // namespace cuda
+
+The function extracts the lower bitfield of size ``width`` from ``source`` and inserts it into ``dest`` at position ``start``.
+
+**Parameters**
+
+- ``dest``:   The value to insert the bitfield into.
+- ``source``: The value from which extract the bitfield.
+- ``start``:  Initial position of the bitfield.
+- ``width``:  Width of the bitfield.
+
+**Return value**
+
+- ``((source << start) & mask) | (dest & ~mask)``, where ``mask`` is a bitmask of width ``width`` at position ``start``.
+
+**Constraints**
+
+- ``T`` is an unsigned integer type.
+
+**Preconditions**
+
+- ``start >= 0 && start <= num_bits(T)``.
+- ``width >= 0 && width <= num_bits(T)``.
+- ``start + width <= num_bits(T)``.
+
+**Performance considerations**
+
+The function performs the following operations in CUDA for ``uint8_t``, ``uint16_t``, ``uint32_t``:
+
+- ``SM < 70``: ``BFI``.
+- ``SM >= 70``: ``BMSK``, bitwise operation x5.
+
+.. note::
+
+    When the input values are run-time values that the compiler can resolve at compile-time, e.g. an index of a loop with a fixed number of iterations, using the function could not be optimal.
+
+.. note::
+
+    GCC <= 8 uses a slow path with more instructions even in CUDA.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/bit>
+    #include <cuda/std/cassert>
+
+    __global__ void bitfield_insert_kernel() {
+        assert(cuda::bitfield_insert(0u, 0xFFFFu, 0, 4) == 0b1111);
+        assert(cuda::bitfield_insert(0u, 0xFFFFu, 3, 4) == 0b1111000);
+        assert(cuda::bitfield_insert(1u, 0xFFFFu, 3, 4) == 0b1111001);
+    }
+
+    int main() {
+        bitfield_insert_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/4Thzz516M>`__

cccl_upstream/docs/libcudacxx/extended_api/bit/bitmask.rst

@@ -0,0 +1,74 @@
+.. _libcudacxx-extended-api-bit-bitmask:
+
+``cuda::bitmask``
+=================
+
+Defined in the ``<cuda/bit>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T = uint32_t>
+   [[nodiscard]] __host__ __device__ constexpr
+   T bitmask(int start, int width) noexcept;
+
+   } // namespace cuda
+
+The function generates a bitmask of size ``width`` starting at position ``start``.
+
+**Parameters**
+
+- ``start``: starting position of the bitmask.
+- ``width``: width of the bitmask.
+
+**Return value**
+
+- Bitmask of size ``width`` starting at position``start``.
+
+**Constraints**
+
+- ``T`` is an unsigned integral type.
+
+**Preconditions**
+
+- ``start >= 0 && start <= num_bits(T)``.
+- ``width >= 0 && width <= num_bits(T)``.
+- ``start + width <= num_bits(T)``.
+
+**Performance considerations**
+
+The function performs the following operations in device code:
+
+- ``uint8_t``, ``uint16_t``, ``uint32_t``: ``BMSK``.
+- ``uint64_t``: ``SHL`` x4, ``ADD`` x2.
+
+.. note::
+
+    When the input values are run-time values that the compiler can resolve at compile-time, e.g. an index of a loop with a fixed number of iterations, using the function could not be optimal.
+
+.. note::
+
+    GCC <= 8 uses a slow path with more instructions even in CUDA.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/bit>
+    #include <cuda/std/cassert>
+    #include <cuda/std/cstdint>
+
+    __global__ void bitmask_kernel() {
+        assert(cuda::bitmask(2, 4) == 0b111100u);
+        assert(cuda::bitmask<uint64_t>(1, 3) == uint64_t{0b1110});
+    }
+
+    int main() {
+        bitmask_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/habGohz7T>`__

cccl_upstream/docs/libcudacxx/extended_api/exceptions.rst

@@ -0,0 +1,34 @@
+.. _libcudacxx-extended-api-exceptions:
+
+Exception Handling
+==================
+
+Standard C++ exception handling (``try``, ``catch``, ``throw``) is not supported in CUDA device code, while it is enabled by default in host code.
+
+**Device code**
+
+``libcu++`` maps exceptions to ``cuda::std::terminate()`` calls in device code, which translates to ``__trap()`` and terminates the kernel.
+
+**Host code**
+
+``libcu++`` allows users to manually disable exceptions in host code in two ways:
+
+- By defining ``CCCL_DISABLE_EXCEPTIONS`` before including any library headers.
+- By compiling with ``-fno-exceptions`` compiler flag with ``gcc`` or ``clang``, or ``/EH-`` compiler flag with ``msvc``.
+
+If exceptions are disabled, a ``throw`` exception is translated into a `cuda::std::terminate() <https://en.cppreference.com/w/cpp/error/terminate.html>`__ call, which terminates the program.
+
+``cuda::cuda_error``
+--------------------
+
+Exception class thrown when a CUDA error is encountered. It inherits from ``std::runtime_error``.
+
+.. code-block:: cpp
+
+    class cuda_error : public std::runtime_error
+    {
+    public:
+        cuda_error(cudaError_t status, const char* msg);
+
+        cudaError_t status() const noexcept;
+    };

cccl_upstream/docs/libcudacxx/extended_api/execution_model.rst

@@ -0,0 +1,296 @@
+.. _libcudacxx-extended-api-execution-model:
+
+Execution model
+===============
+
+CUDA C++ aims to provide `parallel forward progress [intro.progress.9] <https://eel.is/c++draft/intro.progress#9>`__
+for all device threads of execution, facilitating the parallelization of pre-existing C++ applications with CUDA C++.
+
+.. dropdown:: `[intro.progress] <https://eel.is/c++draft/intro.progress>`__
+
+    - `[intro.progress.7] <https://eel.is/c++draft/intro.progress#7>`__: For a thread of execution
+      providing `concurrent forward progress guarantees <https://eel.is/c++draft/intro.progress#def:concurrent_forward_progress_guarantees>`__,
+      the implementation ensures that the thread will eventually make progress for as long as it has not terminated.
+
+      [Note 5: This applies regardless of whether or not other threads of execution (if any) have been or are making progress.
+      To eventually fulfill this requirement means that this will happen in an unspecified but finite amount of time. — end note]
+
+    - `[intro.progress.9] <https://eel.is/c++draft/intro.progress>`__: For a thread of execution providing
+      `parallel forward progress guarantees <https://eel.is/c++draft/intro.progress#9>`__, the implementation is not required to ensure that
+      the thread will eventually make progress if it has not yet executed any execution step; once this thread has executed a step,
+      it provides concurrent forward progress guarantees.
+
+        [Note 6: This does not specify a requirement for when to start this thread of execution, which will typically be specified by the entity
+        that creates this thread of execution. For example, a thread of execution that provides concurrent forward progress guarantees and executes
+        tasks from a set of tasks in an arbitrary order, one after the other, satisfies the requirements of parallel forward progress for these
+        tasks. — end note]
+
+
+The CUDA C++ Programming Language is an extension of the C++ Programming Language.
+This section documents the modifications and extensions to the `[intro.progress] <https://eel.is/c++draft/intro.progress>`__ section of the current `ISO International Standard ISO/IEC 14882 – Programming Language C++ <https://eel.is/c++draft/>`__ draft.
+Modified sections are called out explicitly and their diff is shown in **bold**.
+All other sections are additions.
+
+.. _libcudacxx-extended-api-execution-model-host-threads:
+
+Host threads
+------------
+
+The forward progress provided by threads of execution created by the host implementation to
+execute `main <https://en.cppreference.com/w/cpp/language/main_function>`__, `std::thread <https://en.cppreference.com/w/cpp/thread/thread>`__,
+and `std::jthread <https://en.cppreference.com/w/cpp/thread/jthread>`__ is implementation-defined behavior of the host
+implementation `[intro.progress] <https://eel.is/c++draft/intro.progress>`__.
+General-purpose host implementations should provide concurrent forward progress.
+
+If the host implementation provides `concurrent forward progress [intro.progress.7] <https://eel.is/c++draft/intro.progress#7>`__,
+then CUDA C++ provides `parallel forward progress [intro.progress.9] <https://eel.is/c++draft/intro.progress#9>`__ for device threads.
+
+
+.. _libcudacxx-extended-api-execution-model-device-threads:
+
+Device threads
+--------------
+
+Once a device thread makes progress:
+
+- If it is part of a `Cooperative Grid <https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__EXECUTION.html#group__CUDART__EXECUTION_1g504b94170f83285c71031be6d5d15f73>`__,
+  all device threads in its grid shall eventually make progress.
+- Otherwise, all device threads in its `thread-block cluster <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#thread-block-clusters>`__
+  shall eventually make progress.
+
+    [Note: Threads in other thread-block clusters are not guaranteed to eventually make progress. - end note.]
+
+    [Note: This implies that all device threads within its thread block shall eventually make progress. - end note.]
+
+
+Modify `[intro.progress.1] <https://eel.is/c++draft/intro.progress>`__ as follows (modifications in **bold**):
+
+The implementation may assume that any **host** thread will eventually do one of the following:
+
+    1. terminate,
+    2. invoke the function `std::this_thread::yield <https://en.cppreference.com/w/cpp/thread/yield>`__ (`[thread.thread.this] <http://eel.is/c++draft/thread.thread.this>`__),
+    3. make a call to a library I/O function,
+    4. perform an access through a volatile glvalue,
+    5. perform a synchronization operation or an atomic operation, or
+    6. continue execution of a trivial infinite loop (`[stmt.iter.general] <http://eel.is/c++draft/stmt.iter.general>`__).
+
+**The implementation may assume that any device thread will eventually do one of the following:**
+
+    1. **terminate**,
+    2. **make a call to a library I/O function**,
+    3. **perform an access through a volatile glvalue except if the designated object has automatic storage duration, or**
+    4. **perform a synchronization operation or an atomic read operation except if the designated object has automatic storage duration.**
+
+    [Note: Some current limitations of device threads relative to host threads
+    are implementation defects known to us, that we may fix over time.
+    Examples include the undefined behavior that arises from device threads
+    that eventually only perform volatile or atomic operations
+    on automatic storage duration objects.
+    However, other limitations of device threads relative to host threads
+    are intentional choices.  They enable performance optimizations
+    that would not be possible if device threads followed the C++ Standard strictly.
+    For example, providing forward progress to programs
+    that eventually only perform atomic writes or fences
+    would degrade overall performance for little practical benefit. - end note.]
+
+.. dropdown:: Examples of forward progress guarantee differences between host and device threads due to modifications to `[intro.progress.1] <https://eel.is/c++draft/intro.progress#1>`__.
+
+    The following examples refer to the itemized sub-clauses of the implementation assumptions for host and device threads above
+    using "host.threads.<id>" and "device.threads.<id>", respectively.
+
+    .. code-block:: cuda
+        :linenos:
+
+        // Example: Execution.Model.Device.0
+        // Outcome: grid eventually terminates per device.threads.4 because the atomic object does not have automatic storage duration.
+        __global__ void ex0(cuda::atomic_ref<int, cuda::thread_scope_device> atom) {
+            if (threadIdx.x == 0) {
+                while(atom.load(cuda::memory_order_relaxed) == 0);
+            } else if (threadIdx.x == 1) {
+                atom.store(1, cuda::memory_order_relaxed);
+            }
+        }
+
+    .. code-block:: cuda
+        :linenos:
+
+        // Example: Execution.Model.Device.1
+        // Allowed outcome: No thread makes progress because device threads don't support host.threads.2.
+        __global__ void ex1() {
+            while(true) cuda::std::this_thread::yield();
+        }
+
+    .. code-block:: cuda
+        :linenos:
+
+        // Example: Execution.Model.Device.2
+        // Allowed outcome: No thread makes progress because device threads don't support host.threads.4
+        // for objects with automatic storage duration (see exception in device.threads.3).
+        __global__ void ex2() {
+            volatile bool True = true;
+            while(True);
+        }
+
+    .. code-block:: cuda
+        :linenos:
+
+        // Example: Execution.Model.Device.3
+        // Allowed outcome: No thread makes progress because device threads don't support host.threads.5
+        // for objects with automatic storage duration (see exception in device.threads.4).
+        __global__ void ex3() {
+            cuda::atomic<bool, cuda::thread_scope_thread> True = true;
+            while(True.load());
+        }
+
+    .. code-block:: cuda
+        :linenos:
+
+        // Example: Execution.Model.Device.4
+        // Allowed outcome: No thread makes progress because device threads don't support host.thread.6.
+        __global void ex4() {
+            while(true) { /* empty */ }
+        }
+
+.. _libcudacxx-extended-api-execution-model-cuda-apis:
+
+CUDA APIs
+---------
+
+A CUDA API call shall eventually either return or ensure at least one device thread makes progress.
+
+CUDA query functions (e.g. `cudaStreamQuery <https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__STREAM.html#group__CUDART__STREAM_1g2021adeb17905c7ec2a3c1bf125c5435>`__,
+`cudaEventQuery <https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__EVENT.html#group__CUDART__EVENT_1g2bf738909b4a059023537eaa29d8a5b7>`__, etc.) shall not consistently
+return ``cudaErrorNotReady`` without a device thread making progress.
+
+  [Note: The device thread need not be "related" to the API call, e.g., an API operating on one stream or process may ensure progress of a device thread on another stream or process. - end note.]
+
+  [Note: A simple but not sufficient method to test a program for CUDA API Forward Progress conformance is to run them with following environment variables set: ``CUDA_DEVICE_MAX_CONNECTIONS=1 CUDA_LAUNCH_BLOCKING=1``, and then check that the program still terminates.
+  If it does not, the program has a bug.
+  This method is not sufficient because it does not catch all Forward Progress bugs, but it does catch many such bugs. - end note.]
+
+.. dropdown:: Examples of CUDA API forward progress guarantees.
+
+    .. code-block:: cuda
+        :linenos:
+
+        // Example: Execution.Model.API.1
+        // Outcome: if no other device threads (e.g., from other processes) are making progress,
+        // this program terminates and returns cudaSuccess.
+        // Rationale: CUDA guarantees that if the device is empty:
+        // - `cudaDeviceSynchronize` eventually ensures that at least one device-thread makes progress, which implies that eventually `hello_world` grid and one of its device-threads start.
+        // - All thread-block threads eventually start (due to "if a device thread makes progress, all other threads in its thread-block cluster eventually make progress").
+        // - Once all threads in thread-block arrive at `__syncthreads` barrier, all waiting threads are unblocked.
+        // - Therefore all device threads eventually exit the `hello_world`` grid.
+        // - And `cudaDeviceSynchronize`` eventually unblocks.
+        __global__ void hello_world() { __syncthreads(); }
+        int main() {
+            hello_world<<<1,2>>>();
+            return (int)cudaDeviceSynchronize();
+        }
+
+    .. code-block:: cuda
+        :linenos:
+
+        // Example: Execution.Model.API.2
+        // Allowed outcome: eventually, no thread makes progress.
+        // Rationale: the `cudaDeviceSynchronize` API below is only called if a device thread eventually makes progress and sets the flag.
+        // However, CUDA only guarantees that `producer` device thread eventually starts if the synchronization API is called.
+        // Therefore, the host thread may never be unblocked from the flag spin-loop.
+        cuda::atomic<int, cuda::thread_scope_system> flag = 0;
+        __global__ void producer() { flag.store(1); }
+        int main() {
+            cudaHostRegister(&flag, sizeof(flag));
+            producer<<<1,1>>>();
+            while (flag.load() == 0);
+            return cudaDeviceSynchronize();
+        }
+
+    .. code-block:: cuda
+        :linenos:
+
+        // Example: Execution.Model.API.3
+        // Allowed outcome: eventually, no thread makes progress.
+        // Rationale: same as Example.Model.API.2, with the addition that a single CUDA query API call does not guarantee
+        // the device thread eventually starts, only repeated CUDA query API calls do (see Execution.Model.API.4).
+        cuda::atomic<int, cuda::thread_scope_system> flag = 0;
+        __global__ void producer() { flag.store(1); }
+        int main() {
+            cudaHostRegister(&flag, sizeof(flag));
+            producer<<<1,1>>>();
+            (void)cudaStreamQuery(0);
+            while (flag.load() == 0);
+            return cudaDeviceSynchronize();
+        }
+
+    .. code-block:: cuda
+        :linenos:
+
+        // Example: Execution.Model.API.4
+        // Outcome: terminates.
+        // Rationale: same as Execution.Model.API.3, but this example repeatedly calls
+        // a CUDA query API in within the flag spin-loop, which guarantees that the device thread
+        // eventually makes progress.
+        cuda::atomic<int, cuda::thread_scope_system> flag = 0;
+        __global__ void producer() { flag.store(1); }
+        int main() {
+            cudaHostRegister(&flag, sizeof(flag));
+            producer<<<1,1>>>();
+            while (flag.load() == 0) {
+                (void)cudaStreamQuery(0);
+            }
+            return cudaDeviceSynchronize();
+        }
+
+.. _libcudacxx-extended-api-execution-model-cuda-dependencies:
+
+Dependencies
+~~~~~~~~~~~~
+
+A device thread shall not start until all its dependencies have completed.
+
+  [Note: Dependencies that prevent device threads from starting to make progress can be created, for example, via `CUDA Stream Commands <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#streams>`__ .
+  These may include dependencies on the completion of, among others, `CUDA Events <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#events>`__ and `CUDA Kernels <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#kernels>`__ . - end note.]
+
+.. dropdown:: Examples of CUDA API forward progress guarantees due to dependencies
+
+    .. code-block:: cuda
+        :linenos:
+
+        // Example: Execution.Model.Stream.0
+        // Allowed outcome: eventually, no thread makes progress.
+        // Rationale: while CUDA guarantees that one device thread makes progress, since there
+        // is no dependency between `first` and `second`, it does not guarantee which thread,
+        // and therefore it could always pick the device thread from `second`, which then never
+        // unblocks from the spin-loop.
+        // That is, `second` may starve `first`.
+        cuda::atomic<int, cuda::thread_scope_system> flag = 0;
+        __global__ void first() { flag.store(1, cuda::memory_order_relaxed); }
+        __global__ void second() { while(flag.load(cuda::memory_order_relaxed) == 0) {} }
+        int main() {
+            cudaHostRegister(&flag, sizeof(flag));
+            cudaStream_t s0, s1;
+            cudaStreamCreate(&s0);
+            cudaStreamCreate(&s1);
+            first<<<1,1,0,s0>>>();
+            second<<<1,1,0,s1>>>();
+            return cudaDeviceSynchronize();
+        }
+
+    .. code-block:: cuda
+        :linenos:
+
+        // Example: Execution.Model.Stream.1
+        // Outcome: terminates.
+        // Rationale: same as Execution.Model.Stream.0, but this example has a stream dependency
+        // between first and second, which requires CUDA to run the grids in order.
+        cuda::atomic<int, cuda::thread_scope_system> flag = 0;
+        __global__ void first() { flag.store(1, cuda::memory_order_relaxed); }
+        __global__ void second() { while(flag.load(cuda::memory_order_relaxed) == 0) {} }
+        int main() {
+            cudaHostRegister(&flag, sizeof(flag));
+            cudaStream_t s0;
+            cudaStreamCreate(&s0);
+            first<<<1,1,0,s0>>>();
+            second<<<1,1,0,s0>>>();
+            return cudaDeviceSynchronize();
+        }

cccl_upstream/docs/libcudacxx/extended_api/functional.rst

@@ -0,0 +1,62 @@
+.. _libcudacxx-extended-api-functional:
+
+Functional
+----------
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   functional/always_true_false
+   functional/proclaim_return_type
+   functional/maximum_minimum
+   functional/operator_properties
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`cuda::always_true <libcudacxx-extended-api-functional-always-true-false>`
+     - Function object that always returns ``true``
+     - CCCL 3.4.0
+     - CUDA 13.4
+
+   * - :ref:`cuda::always_false <libcudacxx-extended-api-functional-always-true-false>`
+     - Function object that always returns ``false``
+     - CCCL 3.4.0
+     - CUDA 13.4
+
+   * - :ref:`cuda::maximum <libcudacxx-extended-api-functional-maximum-minimum>`
+     - Returns the maximum of two values
+     - CCCL 2.8.0
+     - CUDA 12.9
+
+   * - :ref:`cuda::minimum <libcudacxx-extended-api-functional-maximum-minimum>`
+     - Returns the minimum of two values
+     - CCCL 2.8.0
+     - CUDA 12.9
+
+   * - :ref:`cuda::proclaim_return_type <libcudacxx-extended-api-functional-proclaim-return-type>`
+     - Creates a forwarding call wrapper that proclaims return type
+     - libcu++ 1.9.0 / CCCL 2.0.0
+     - CUDA 11.8
+
+   * - ``cuda::proclaim_copyable_arguments``
+     - Creates a forwarding call wrapper that proclaims that arguments can be freely copied before an invocation of the wrapped callable
+     - CCCL 2.8.0
+     - CUDA 12.9
+
+   * - :ref:`cuda::get_device_address <libcudacxx-extended-api-memory-get-device-address>`
+     - Returns a valid address to a device object
+     - CCCL 2.8.0
+     - CUDA 12.9
+
+   * - :ref:`cuda::is_associative_v <libcudacxx-extended-api-functional-operator-properties>`, :ref:`cuda::is_commutative_v <libcudacxx-extended-api-functional-operator-properties>`, :ref:`cuda::identity_element() <libcudacxx-extended-api-functional-operator-properties>`, :ref:`cuda::absorbing_element() <libcudacxx-extended-api-functional-operator-properties>`
+     - Determines if an operator is associative for a type
+     - CCCL 3.3.0
+     - CUDA 13.3

cccl_upstream/docs/libcudacxx/extended_api/functional/always_true_false.rst

@@ -0,0 +1,44 @@
+.. _libcudacxx-extended-api-functional-always-true-false:
+
+``cuda::always_true`` and ``cuda::always_false``
+================================================
+
+Defined in the header ``<cuda/functional>``.
+
+.. code:: cuda
+
+    struct always_true {
+        template <typename... Ts>
+        [[nodiscard]] __host__ __device__ constexpr bool operator()(Ts&&...) const noexcept;
+    };
+
+    struct always_false {
+        template <typename... Ts>
+        [[nodiscard]] __host__ __device__ constexpr bool operator()(Ts&&...) const noexcept;
+    };
+
+``cuda::always_true`` is a function object that always returns ``true`` regardless of the number and type of arguments
+passed. ``cuda::always_false`` is a function object that always returns ``false`` regardless of the number and type of
+arguments passed.
+
+Both types are empty, trivially copyable, and their ``operator()`` is ``constexpr`` and ``noexcept``.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/functional>
+
+    __global__ void example_kernel() {
+        cuda::always_true  pred_true{};
+        cuda::always_false pred_false{};
+
+        // Returns true regardless of arguments
+        static_assert(pred_true());
+        static_assert(pred_true(1, 2, 3));
+
+        // Returns false regardless of arguments
+        static_assert(!pred_false());
+        static_assert(!pred_false(1, 2, 3));
+    }

cccl_upstream/docs/libcudacxx/extended_api/functional/maximum_minimum.rst

@@ -0,0 +1,74 @@
+.. _libcudacxx-extended-api-functional-maximum-minimum:
+
+``cuda::maximum`` and ``cuda::minimum``
+=======================================
+
+Defined in the header ``<cuda/functional>``.
+
+.. code:: cuda
+
+    template <typename T>
+    struct maximum {
+        [[nodiscard]] __host__ __device__ constexpr
+        T operator()(const T& a, const T& b) const noexcept(/* see below */);
+    };
+
+    template <>
+    struct maximum<void> {
+        template <typename T1, typename T2>
+        [[nodiscard]] __host__ __device__ constexpr
+        cuda::std::common_type_t<T1, T2> operator()(const T1& a, const T2& b) const noexcept(/* see below */);
+    };
+
+    template <typename T>
+    struct minimum {
+        [[nodiscard]] __host__ __device__ constexpr
+        T operator()(const T& a, const T& b) const noexcept(/* see below */);
+    };
+
+    template <>
+    struct minimum<void> {
+        template <typename T1, typename T2>
+        [[nodiscard]] __host__ __device__ constexpr
+        cuda::std::common_type_t<T1, T2> operator()(const T1& a, const T2& b) const noexcept(/* see below */);
+    };
+
+Function objects for performing maximum and minimum operations. The ``operator()`` is ``noexcept`` when the comparison between the values is also ``noexcept``.
+
+.. note::
+
+    Differently from ``std::plus`` and other functional operators, ``cuda::maximum`` and ``cuda::minimum`` specialized for ``void`` returns ``cuda::std::common_type_t`` and not the implicit promotion
+
+Floating-Point Behavior
+-----------------------
+
+For floating-point types (and extended floating-point types), ``cuda::maximum`` uses ``cuda::std::fmax`` and ``cuda::minimum`` uses ``cuda::std::fmin`` instead of the comparison operator, following the ``std::fmax``/``std::fmin`` specification for handling special values such as ``NaN``.
+
+This also makes ``cuda::maximum`` and ``cuda::minimum`` commutative for floating-point types, unlike a plain comparison-based approach.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/functional>
+    #include <cuda/std/cstdint>
+    #include <cstdio>
+    #include <numeric>
+
+    __global__ void maximum_minimum_kernel() {
+        uint16_t v1 = 7;
+        uint16_t v2 = 3;
+        printf("%d\n", cuda::maximum<uint16_t>{}(v1, v2)); // print "7" (uint16_t)
+        printf("%d\n", cuda::minimum{}(v1, v2));           // print "3" (int)
+    }
+
+    int main() {
+        maximum_minimum_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+        int array[] = {3, 7, 5, 2};
+        printf("%d\n", std::accumulate(array, array + 4, 0, cuda::maximum{})); // 7
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/44fdTerre>`_

cccl_upstream/docs/libcudacxx/extended_api/functional/operator_properties.rst

@@ -0,0 +1,340 @@
+.. _libcudacxx-extended-api-functional-operator-properties:
+
+Operator Properties
+===================
+
+Defined in the header ``<cuda/functional>``.
+
+The operator properties traits provide compile-time information about algebraic properties of binary operators.
+These traits are useful for generic algorithms that can apply optimizations based on operator properties,
+such as parallel reductions that can reorder operations for associative operators.
+
+Associativity
+-------------
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <class Op, class T, class Enable = void>
+   inline constexpr bool is_associative_v;
+
+   } // namespace cuda
+
+Determines whether a binary operator ``Op`` is associative for type ``T``, meaning ``op(op(a, b), c) == op(a, op(b, c))``
+for all values ``a``, ``b``, ``c`` of type ``T``. This allows the implementation to reorder operations.
+
+**Supported operators and types:**
+
+.. list-table::
+   :widths: 30 35 35
+   :header-rows: 1
+
+   * - Operator
+     - Integer Types
+     - Floating-Point Types
+   * - ``cuda::std::plus``
+     - ``true``
+     - ``false`` (due to rounding errors)
+   * - ``cuda::std::multiplies``
+     - ``true``
+     - ``false`` (due to rounding errors)
+   * - ``cuda::std::minus``
+     - ``false``
+     - ``false``
+   * - ``cuda::std::divides``
+     - ``false``
+     - ``false``
+   * - ``cuda::std::modulus``
+     - ``false``
+     - N/A
+   * - ``cuda::std::bit_and``
+     - ``true``
+     - N/A
+   * - ``cuda::std::bit_or``
+     - ``true``
+     - N/A
+   * - ``cuda::std::bit_xor``
+     - ``true``
+     - N/A
+   * - ``cuda::std::logical_and``
+     - ``true`` (``bool`` only)
+     - N/A
+   * - ``cuda::std::logical_or``
+     - ``true`` (``bool`` only)
+     - N/A
+   * - ``cuda::minimum``
+     - ``true``
+     - ``true``
+   * - ``cuda::maximum``
+     - ``true``
+     - ``true``
+
+.. note::
+
+   In the strictest sense of the term, the operations of plus and multiplication for integral values may result in undefined behaviour due to overflow. However, in the context of parallel algorithms, they are considered to be associative.
+
+Commutativity
+-------------
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <class Op, class T, class Enable = void>
+   inline constexpr bool is_commutative_v;
+
+   } // namespace cuda
+
+Determines whether a binary operator ``Op`` is commutative for type ``T``, meaning ``op(a, b) == op(b, a)``
+for all values ``a``, ``b`` of type ``T``.
+
+**Supported operators and types:**
+
+.. list-table::
+   :widths: 30 35 35
+   :header-rows: 1
+
+   * - Operator
+     - Integer Types
+     - Floating-Point Types
+   * - ``cuda::std::plus``
+     - ``true``
+     - ``true``
+   * - ``cuda::std::multiplies``
+     - ``true``
+     - ``true``
+   * - ``cuda::std::minus``
+     - ``false``
+     - ``false``
+   * - ``cuda::std::divides``
+     - ``false``
+     - ``false``
+   * - ``cuda::std::modulus``
+     - ``false``
+     - N/A
+   * - ``cuda::std::bit_and``
+     - ``true``
+     - N/A
+   * - ``cuda::std::bit_or``
+     - ``true``
+     - N/A
+   * - ``cuda::std::bit_xor``
+     - ``true``
+     - N/A
+   * - ``cuda::std::logical_and``
+     - ``true`` (``bool`` only)
+     - N/A
+   * - ``cuda::std::logical_or``
+     - ``true`` (``bool`` only)
+     - N/A
+   * - ``cuda::minimum``
+     - ``true``
+     - ``true``
+   * - ``cuda::maximum``
+     - ``true``
+     - ``true``
+
+Identity Element
+----------------
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <class Op, class T>
+   __host__ __device__
+   constexpr auto identity_element() noexcept;
+
+   template <class Op, class T, class Enable = void>
+   inline constexpr bool has_identity_element_v;
+
+   } // namespace cuda
+
+Provides the identity element for operator ``Op`` and type ``T``. The identity element ``e`` satisfies
+``op(e, x) == op(x, e) == x`` for all values ``x`` of type ``T``.
+
+The function ``identity_element<Op, T>()`` returns the identity element at compile time. If no identity element
+exists for the given operator and type combination, it returns an internal sentinel type.
+
+``has_identity_element_v`` evaluates to ``true`` if an identity element is defined for the given operator and type.
+
+**Identity elements by operator:**
+
+.. list-table::
+   :widths: 30 35 35
+   :header-rows: 1
+
+   * - Operator
+     - Integer Types
+     - Floating-Point Types
+   * - ``cuda::std::plus``
+     - ``T{0}``
+     - ``-0.0`` (negative zero)
+   * - ``cuda::std::multiplies``
+     - ``T{1}``
+     - ``1.0``
+   * - ``cuda::std::bit_and``
+     - ``~T{0}`` (all bits set)
+     - N/A
+   * - ``cuda::std::bit_or``
+     - ``T{0}``
+     - N/A
+   * - ``cuda::std::bit_xor``
+     - ``T{0}``
+     - N/A
+   * - ``cuda::std::logical_and``
+     - ``true`` (``bool`` only)
+     - N/A
+   * - ``cuda::std::logical_or``
+     - ``false`` (``bool`` only)
+     - N/A
+   * - ``cuda::minimum``
+     - ``numeric_limits<T>::max()``
+     - ``+infinity``
+   * - ``cuda::maximum``
+     - ``numeric_limits<T>::lowest()``
+     - ``-infinity``
+
+.. note::
+
+   For floating-point ``plus``, the identity element is negative zero (``-0.0``) rather than positive zero.
+   This preserves the sign when adding: ``-0.0 + (-0.0) == -0.0``.
+
+.. note::
+
+   ``cuda::std::minus``, ``cuda::std::divides``, and ``cuda::std::modulus`` do not have identity elements.
+
+Absorbing Element
+-----------------
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <class Op, class T>
+   __host__ __device__
+   constexpr auto absorbing_element() noexcept;
+
+   template <class Op, class T, class Enable = void>
+   inline constexpr bool has_absorbing_element_v;
+
+   } // namespace cuda
+
+Provides the absorbing (annihilating) element for operator ``Op`` and type ``T``. The absorbing element ``z`` satisfies
+``op(z, x) == op(x, z) == z`` for all values ``x`` of type ``T``.
+
+The function ``absorbing_element<Op, T>()`` returns the absorbing element at compile time. If no absorbing element
+exists for the given operator and type combination, it returns an internal sentinel type.
+
+``has_absorbing_element_v`` evaluates to ``true`` if an absorbing element is defined for the given operator and type.
+
+**Absorbing elements by operator:**
+
+.. list-table::
+   :widths: 30 35 35
+   :header-rows: 1
+
+   * - Operator
+     - Integer Types
+     - Floating-Point Types
+   * - ``cuda::std::multiplies``
+     - ``T{0}``
+     - N/A (see note)
+   * - ``cuda::std::bit_and``
+     - ``T{0}``
+     - N/A
+   * - ``cuda::std::bit_or``
+     - ``~T{0}`` (all bits set)
+     - N/A
+   * - ``cuda::std::logical_and``
+     - ``false`` (``bool`` only)
+     - N/A
+   * - ``cuda::std::logical_or``
+     - ``true`` (``bool`` only)
+     - N/A
+   * - ``cuda::minimum``
+     - ``numeric_limits<T>::lowest()``
+     - ``-infinity``
+   * - ``cuda::maximum``
+     - ``numeric_limits<T>::max()``
+     - ``+infinity``
+
+.. note::
+
+   Floating-point ``multiplies`` does not have an absorbing element because:
+
+   - ``0 * NaN = NaN`` (not ``0``)
+   - ``0 * infinity = NaN`` (not ``0``)
+   - ``(-1) * (+0) = -0`` (not ``+0``)
+
+.. note::
+
+   ``cuda::std::plus``, ``cuda::std::minus``, ``cuda::std::divides``, ``cuda::std::modulus``, and ``cuda::std::bit_xor``
+   do not have absorbing elements.
+
+Supported Types
+---------------
+
+The functionality supports all integer and floating-point types, including extended floating-point types.
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/functional>
+   #include <cuda/std/cstdio>
+
+   template <class Op, class T>
+   __host__ __device__ void print_properties() {
+        printf("Associative:   %s\n", cuda::is_associative_v<Op, T> ? "yes" : "no");
+        printf("Commutative:   %s\n", cuda::is_commutative_v<Op, T> ? "yes" : "no");
+        printf("Has identity:  %s\n", cuda::has_identity_element_v<Op, T> ? "yes" : "no");
+        printf("Has absorbing: %s\n", cuda::has_absorbing_element_v<Op, T> ? "yes" : "no");
+
+        if constexpr (cuda::has_identity_element_v<Op, T>) {
+            printf("Identity element exists\n");
+        }
+        if constexpr (cuda::has_absorbing_element_v<Op, T>) {
+            printf("Absorbing element exists\n");
+        }
+   }
+
+   __global__ void example_kernel() {
+        // Integer plus: associative, commutative, identity=0, no absorbing
+        print_properties<cuda::std::plus<int>, int>();
+
+        // Integer plus: NOT associative, commutative, identity=-0.0f, no absorbing
+        print_properties<cuda::std::plus<float>, float>();
+
+        // identity element
+        constexpr int sum_identity = cuda::identity_element<cuda::std::plus<int>, int>();       // 0
+        constexpr int mul_identity = cuda::identity_element<cuda::std::multiplies<int>, int>(); // 1
+   }
+
+Customization
+-------------
+
+Users can extend the operator properties traits to support custom operators or custom types by specializing
+the internal variable templates. The following specializations are available:
+
+.. code:: cuda
+
+   template <>
+   inline constexpr bool cuda::is_associative_v<MyOperator, MyType> = true;
+
+   template <>
+   inline constexpr bool cuda::is_commutative_v<MyOperator, MyType> = true;
+
+   template <>
+   __host__ __device__
+   constexpr auto cuda::identity_element<MyOperator, MyType>() noexcept {
+        return /* MyType identity element */;
+   }
+
+   template <>
+   __host__ __device__
+   constexpr auto cuda::absorbing_element<MyOperator, MyType>() noexcept {
+       return /* MyType absorbing element */;
+   }

cccl_upstream/docs/libcudacxx/extended_api/functional/proclaim_return_type.rst

@@ -0,0 +1,64 @@
+.. _libcudacxx-extended-api-functional-proclaim-return-type:
+
+``cuda::proclaim_return_type``
+==============================
+
+Defined in the header ``<cuda/functional>``:
+
+.. code:: cuda
+
+   template <class Ret, class Fn>
+   __host__ __device__
+   unspecified<Ret, Fn> proclaim_return_type(Fn&& fn) {
+     return unspecified<Ret, Fn>{fn};
+   }
+
+``cuda::proclaim_return_type`` creates a forwarding call wrapper that uses ``Ret`` as a return type.
+The wrapper is useful in the case of extended device lambdas since an attempt to determine the return type of
+their ``operator()`` function may work incorrectly in host code.
+
+Template Parameters
+-------------------
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``Ret``
+     - Return type that's being proclaimed
+   * - ``Fn``
+     - Callable object type that's being wrapped
+
+Parameters
+----------
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``fn``
+     - Callable object that's being wrapped
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/functional>
+
+   template <class T, class Fn>
+   __global__ void example_kernel(T *out, Fn fn) {
+     *out = fn();
+   }
+
+   __host__ void example() {
+     auto fn = cuda::proclaim_return_type<char>([] __device__ () { return 'd'; });
+     using rt = cuda::std::invoke_result_t<decltype(fn)>;
+
+     rt* out {};
+     cudaMalloc(&out, sizeof(rt));
+
+     example_kernel<<<1, 1>>>(out, fn);
+
+     // ...
+   }

cccl_upstream/docs/libcudacxx/extended_api/iterators.rst

@@ -0,0 +1,15 @@
+.. _libcudacxx-extended-api-iterators:
+
+Fancy Iterators
+---------------
+
+CCCL provides a set of fancy iterators that originate from ``Thrust``.
+They allow the user to avoid storing data needlessly in memory and fuse multiple iterations into a single run.
+
+.. toctree::
+   :glob:
+   :maxdepth: 1
+
+   ../api/class*__iterator
+   ../api/class*zip__function
+   ../api/group__iterators*

cccl_upstream/docs/libcudacxx/extended_api/macros.rst

@@ -0,0 +1,34 @@
+.. _libcudacxx-extended-api-macros:
+
+======
+Macros
+======
+
+CCCL provides a set of convenience macros for detecting various system and compile-time
+properties via the preprocessor. These macros are available when any CCCL header is
+included, and do not require including a specific header file.
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Macro**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - .. toctree::
+          :maxdepth: 1
+
+          ../api/macro_cccl_host_arch
+     - Detecting the current host architecture.
+     - CCCL 3.5.0
+     - CUDA 13.5
+
+   * - .. toctree::
+          :maxdepth: 1
+
+          ../api/macro_cccl_os
+     - Detecting the current operating system.
+     - CCCL 3.4.0
+     - CUDA 13.3

cccl_upstream/docs/libcudacxx/extended_api/math.rst

@@ -0,0 +1,115 @@
+.. _libcudacxx-extended-api-math:
+
+Math
+====
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   math/ceil_div
+   math/round_up
+   math/round_down
+   math/ilog
+   math/ipow
+   math/pow2
+   math/isqrt
+   math/neg
+   math/uabs
+   math/fast_mod_div
+   math/mul_hi
+   math/sincos
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`ceil_div <libcudacxx-extended-api-math-ceil-div>`
+     - Ceiling division
+     - CCCL 2.7.0
+     - CUDA 12.8
+
+   * - :ref:`round_up <libcudacxx-extended-api-math-round-up>`
+     - Round up to the next multiple
+     - CCCL 3.0.0
+     - CUDA 13.0
+
+   * - :ref:`round_down <libcudacxx-extended-api-math-round-down>`
+     - Round down to the previous multiple
+     - CCCL 3.0.0
+     - CUDA 13.0
+
+   * - :ref:`ilog2 <libcudacxx-extended-api-math-ilog>`
+     - Integer logarithm to the base 2
+     - CCCL 3.0.0
+     - CUDA 13.0
+
+   * - :ref:`ceil_ilog2 <libcudacxx-extended-api-math-ilog>`
+     - Integer logarithm to the base 2, rounded up
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`ilog10 <libcudacxx-extended-api-math-ilog>`
+     - Integer logarithm to the base 10
+     - CCCL 3.0.0
+     - CUDA 13.0
+
+   * - :ref:`ceil_ilog10 <libcudacxx-extended-api-math-ilog>`
+     - Integer logarithm to the base 10, rounded up
+     - CCCL 3.5.0
+     - CUDA 13.5
+
+   * - :ref:`ipow <libcudacxx-extended-api-math-ipow>`
+     - Integer power
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`is_power_of_two <libcudacxx-extended-api-math-pow2>`
+     - If the value is a power of two
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`isqrt <libcudacxx-extended-api-math-isqrt>`
+     - Integer square root
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`neg <libcudacxx-extended-api-math-neg>`
+     - Integer negation
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`next_power_of_two <libcudacxx-extended-api-math-pow2>`
+     - Next power of two
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`prev_power_of_two <libcudacxx-extended-api-math-pow2>`
+     - Previous power of two
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`uabs <libcudacxx-extended-api-math-uabs>`
+     - Unsigned absolute value
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`fast_mod_div <libcudacxx-extended-api-math-fast-mod-div>`
+     - Fast Modulo/Division
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`mul_hi <libcudacxx-extended-api-math-mul-hi>`
+     - Most significant half of the product
+     - CCCL 3.2.0
+     - CUDA 13.2
+
+   * - :ref:`sincos <libcudacxx-extended-api-math-sincos>`
+     - Computes sine and cosine of a value at the same time.
+     - CCCL 3.3.0
+     - CUDA 13.3

cccl_upstream/docs/libcudacxx/extended_api/math/ceil_div.rst

@@ -0,0 +1,77 @@
+.. _libcudacxx-extended-api-math-ceil-div:
+
+``cuda::ceil_div``
+==================
+
+Defined in the ``<cuda/cmath>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T, typename U>
+   [[nodiscard]] __host__ __device__ constexpr
+   cuda::std::common_type_t<T, U> ceil_div(T value, U divisor) noexcept;
+
+   } // namespace cuda
+
+The function computes the ceiling division between two integral or enumerator values :math:`ceil(\frac{value}{base\_multiple})`.
+
+**Parameters**
+
+- ``value``: The value to be divided.
+- ``divisor``: The divisor.
+
+**Return value**
+
+Divides ``value`` by ``divisor``. If ``value`` is not a multiple of ``divisor`` rounds the result up to the next integer value.
+
+**Constraints**
+
+- ``T`` and ``U`` are integer types or enumerators.
+
+**Preconditions**
+
+- ``value >= 0``
+- ``divisor > 0``
+
+**Performance considerations**
+
+- The function computes ``(value + divisor - 1) / divisor`` when the common type is a signed integer.
+- The function computes ``min(value, 1 + ((value - 1) / divisor)`` when the common type is an unsigned integer in CUDA, which generates less instructions than ``(value / divisor) + ((value / divisor) * divisor != value)``, especially for 64-bit types.
+
+Example
+-------
+
+This API is very useful for determining the *number of thread blocks* required to process a fixed amount of work, given a fixed number of threads per block:
+
+.. code:: cuda
+
+    #include <cuda/cmath>
+    #include <cuda/std/span>
+    #include <thrust/device_vector.h>
+
+    __global__ void vector_scale_kernel(cuda::std::span<float> span, float scale) {
+        int index = blockIdx.x * blockDim.x + threadIdx.x;
+        if (index < span.size())
+            span[index] *= scale;
+    }
+
+    int main() {
+        int   num_items = 100'000;
+        float scale = 2.f;
+        thrust::device_vector<float> d_vector(num_items, 1.f);
+        // Given a fixed number of threads per block...
+        constexpr int threads_per_block = 256;
+        // ...dividing some "n" by "threads_per_block" may lead to a remainder,
+        // requiring the kernel to be launched with an extra thread block to handle it.
+        auto num_thread_blocks = cuda::ceil_div(num_items, threads_per_block);
+        auto d_ptr             = thrust::raw_pointer_cast(d_vector.data());
+        cuda::std::span<float> d_span(d_ptr, num_items);
+
+        vector_scale_kernel<<<num_thread_blocks, threads_per_block>>>(d_span, scale);
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/dj9a5137z>`__

cccl_upstream/docs/libcudacxx/extended_api/math/fast_mod_div.rst

@@ -0,0 +1,92 @@
+.. _libcudacxx-extended-api-math-fast-mod-div:
+
+``cuda::fast_mod_div``
+======================
+
+Defined in the ``<cuda/cmath>`` header.
+
+.. code:: cuda
+
+    namespace cuda {
+
+    template <typename T, bool DivisorIsNeverOne = false>
+    class fast_mod_div {
+    public:
+        fast_mod_div() = delete;
+
+        __host__ __device__
+        explicit fast_mod_div(T divisor) noexcept;
+
+        template <typename U>
+        [[nodiscard]] __host__ __device__ friend
+        cuda:::std::common_type_t<T, U> operator/(U dividend, fast_mod_div<T> divisor) noexcept;
+
+        template <typename U>
+        [[nodiscard]] __host__ __device__ friend
+        cuda:::std::common_type_t<T, U> operator%(U dividend, fast_mod_div<T> divisor) noexcept;
+
+        [[nodiscard]] __host__ __device__
+        operator T() const noexcept;
+    };
+
+    } // namespace cuda
+
+.. code:: cuda
+
+    namespace cuda {
+
+    template <typename T, typename U>
+    [[nodiscard]] __host__ __device__
+    cuda::std::pair<T, U> div(T dividend, fast_mod_div<U> divisor) noexcept;
+
+    } // namespace cuda
+
+The class ``fast_mod_div`` is used to pre-compute the modulo and division of an integer value, to be used in a second stage for efficiency: :math:`floor\left(\frac{dividend}{divisor}\right)`.
+
+**Parameters**
+
+- ``divisor``:  The divisor.
+- ``dividend``: The dividend.
+- ``DivisorIsNeverOne``: Indicates that ``divisor != 1`` and skips one comparison in the second stage.
+
+**Constraints**
+
+- ``T`` and ``U`` are integer types.
+- ``max_value(dividend type) <= max_value(divisor type)``.
+
+**Preconditions**
+
+- ``divisor > 0``.
+- ``dividend >= 0``.
+- ``divisor > 1`` if ``DivisorIsNeverOne == true``.
+
+**Performance considerations**
+
+- ``fast_mod_div`` needs to be initialized on the host and executed on the device for optimal performance.
+- ``T`` signed type ensures the best performance. ``T == int`` translates to ``SEL``, ``IMAD``, and x2 ``SHF`` instructions.
+- Larger types (> 32-bits) are slower than smaller types.
+- ``DivisorIsNeverOne == true`` can be used to skip one comparison.
+- ``__builtin_assume(dividend != cuda::std::numeric_limits<U>::max())`` can be used to skip one comparison with unsigned values.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/cmath>
+    #include <cuda/std/cassert>
+
+    __global__ void div_kernel(cuda::fast_mod_div<int> divisor) {
+        assert(45 / divisor == 2);
+        assert(45 % divisor == 5);
+        assert((cuda::div(45, divisor) == cuda::std::pair{2, 5}));
+    }
+
+    int main() {
+        cuda::fast_mod_div<int> divisor(20);
+        div_kernel<<<1, 1>>>(divisor);
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/fM7E9v9aP>`__

cccl_upstream/docs/libcudacxx/extended_api/math/ilog.rst

@@ -0,0 +1,83 @@
+.. _libcudacxx-extended-api-math-ilog:
+
+``cuda::ilog2``, ``cuda::ceil_ilog2``, ``cuda::ilog10``, and ``cuda::ceil_ilog10``
+=================================================================================
+
+Defined in the ``<cuda/cmath>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ __tile__ constexpr
+   int ilog2(T value) noexcept;
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ __tile__ constexpr
+   int ceil_ilog2(T value) noexcept;
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ __tile__ constexpr
+   int ilog10(T value) noexcept;
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ __tile__ constexpr
+   int ceil_ilog10(T value) noexcept;
+
+   } // namespace cuda
+
+The functions compute the logarithm to the base 2 and 10 of an integer value.
+
+**Parameters**
+
+- ``value``: The input value.
+
+**Return value**
+
+- ``ilog2``, ``ceil_ilog2``: The logarithm to the base 2, rounded down and up to the nearest integer respectively.
+- ``ilog10``, ``ceil_ilog10``: The logarithm to the base 10, rounded down and up to the nearest integer respectively.
+
+**Constraints**
+
+- ``T`` is an integer type.
+
+**Preconditions**
+
+- ``value > 0``
+
+**Performance considerations**
+
+The functions perform the following operations in device code:
+
+- ``ilog2``: ``FLO``
+- ``ceil_ilog2``: ``FLO``, ``POPC``, ``ADD``, comparison
+- ``ilog10``: ``FLO``, ``FMUL``, ``F2I``, constant memory lookup, ``SEL`` + ``IADD`` only if ``T == uint32_t`` or ``T == __uint128_t``
+- ``ceil_ilog10``: ``ilog10`` with an additional comparison, subtraction, and ``IADD``
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/cmath>
+    #include <cuda/std/cassert>
+
+    __global__ void ilog_kernel() {
+        assert(cuda::ilog2(20) == 4);
+        assert(cuda::ceil_ilog2(20) == 5);
+        assert(cuda::ilog2(32) == 5);
+        assert(cuda::ceil_ilog2(32) == 5);
+        assert(cuda::ilog10(100) == 2);
+        assert(cuda::ilog10(2000) == 3);
+        assert(cuda::ceil_ilog10(100) == 2);
+        assert(cuda::ceil_ilog10(2000) == 4);
+    }
+
+    int main() {
+        ilog_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/7W3WaGd3c>`__

cccl_upstream/docs/libcudacxx/extended_api/math/ipow.rst

@@ -0,0 +1,60 @@
+.. _libcudacxx-extended-api-math-ipow:
+
+``cuda::ipow``
+====================================
+
+Defined in the ``<cuda/cmath>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T, typename E>
+   [[nodiscard]] __host__ __device__ constexpr
+   T ipow(T base, E exp) noexcept;
+
+   } // namespace cuda
+
+The function computes the integer ``base`` raised to the power of ``exp``.
+
+**Parameters**
+
+- ``base``: The base value.
+- ``exp``: The exponent value.
+
+**Return value**
+
+- The result of raising ``base`` to the power of ``exp``. If ``exp`` is negative, the result is 0.
+
+**Constraints**
+
+- ``T`` is an integer type.
+- ``E`` is an integer type.
+
+**Preconditions**
+
+- if ``base`` is 0, then ``exp`` must be non-negative.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/cmath>
+    #include <cuda/std/cassert>
+
+    __global__ void ipow_kernel() {
+        assert(cuda::ipow(0, 0) == 1);
+        assert(cuda::ipow(2, 2) == 4);
+        assert(cuda::ipow(99, 1) == 99);
+        assert(cuda::ipow(4, 7) == 16384);
+        assert(cuda::ipow(-1, 3) == -1);
+        assert(cuda::ipow(23, -1) == 0);
+    }
+
+    int main() {
+        ipow_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/TMacWvz8v>`__

cccl_upstream/docs/libcudacxx/extended_api/math/isqrt.rst

@@ -0,0 +1,57 @@
+.. _libcudacxx-extended-api-math-isqrt:
+
+``cuda::isqrt``
+====================================
+
+Defined in the ``<cuda/cmath>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ constexpr
+   T isqrt(T value) noexcept;
+
+   } // namespace cuda
+
+The function computes the integer square root of the input value rounded down.
+
+**Parameters**
+
+- ``value``: The input value.
+
+**Return value**
+
+- The square root value of the input value rounded down.
+
+**Constraints**
+
+- ``T`` is an integer type.
+
+**Preconditions**
+
+- ``value`` is non-negative.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/cmath>
+    #include <cuda/std/cassert>
+
+    __global__ void isqrt_kernel() {
+        assert(cuda::isqrt(1) == 1);
+        assert(cuda::isqrt(4) == 2);
+        assert(cuda::isqrt(42) == 6);
+        assert(cuda::isqrt(99) == 9);
+        assert(cuda::isqrt(100) == 10);
+    }
+
+    int main() {
+        isqrt_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/xPcj35dq6>`__

cccl_upstream/docs/libcudacxx/extended_api/math/mul_hi.rst

@@ -0,0 +1,64 @@
+.. _libcudacxx-extended-api-math-mul-hi:
+
+``cuda::mul_hi``
+================
+
+Defined in ``<cuda/cmath>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ constexpr
+   T mul_hi(T lhs, T rhs) noexcept;
+
+   } // namespace cuda
+
+Computes the most significant half of the bits of the product of two non-negative integers ``lhs`` and ``rhs``.
+
+**Parameters**
+
+- ``lhs``: First multiplicand.
+- ``rhs``: Second multiplicand.
+
+**Return value**
+
+- The most significant half of ``lhs * rhs`` returned as ``T``.
+
+**Constraints**
+
+- ``T`` is an integer type.
+
+**Remarks**
+
+- Uses ``__mulhi``, ``__umulhi``, ``__mul64hi``, ``__umul64hi`` intrinsics on device when available.
+- Uses ``__mulh``, ``__umulh`` intrinsics on Windows host code when available.
+- Uses a double-width intermediate type when possible.
+- Relies on a manual decomposition fallback when 128-bit intermediates are unavailable for 64-bit integers.
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/cmath>
+   #include <cuda/std/cassert>
+   #include <cuda/std/cstdint>
+
+   __global__ void mul_hi_kernel()
+   {
+       uint32_t lhs       = 0xABCD1234;
+       uint32_t rhs       = 1 << 16; // 2^16
+       uint32_t high_half = cuda::mul_hi(lhs, rhs);
+       assert(high_half == 0xAB);
+   }
+
+   int main()
+   {
+       mul_hi_kernel<<<1, 1>>>();
+       cudaDeviceSynchronize();
+       return 0;
+   }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/64r6zT9Wq>`__

cccl_upstream/docs/libcudacxx/extended_api/math/neg.rst

@@ -0,0 +1,57 @@
+.. _libcudacxx-extended-api-math-neg:
+
+``cuda::neg``
+====================================
+
+Defined in the ``<cuda/cmath>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ constexpr
+   T neg(T value) noexcept;
+
+   } // namespace cuda
+
+The function computes the negation of the input value accepting both signed and unsigned integer types. It doesn't emit any warnings for signed integer overflow and applying ``-`` to unsigned integer types.
+
+**Parameters**
+
+- ``value``: The input value.
+
+**Return value**
+
+- The negated value of the input value.
+
+**Constraints**
+
+- ``T`` is an integer type.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/cmath>
+    #include <cuda/std/cassert>
+    #include <cuda/std/limits>
+
+    __global__ void neg_kernel() {
+        using cuda::std::numeric_limits;
+
+        assert(cuda::neg(1) == -1);
+        assert(cuda::neg(20) == -20);
+        assert(cuda::neg(127u) == 4294967169u);
+        assert(cuda::neg(-127) == 127);
+        assert(cuda::neg(cuda::std::numeric_limits<int>::min()) == cuda::std::numeric_limits<int>::min());
+    }
+
+    int main() {
+        neg_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/K3zcE9zqn>`__

cccl_upstream/docs/libcudacxx/extended_api/math/pow2.rst

@@ -0,0 +1,70 @@
+.. _libcudacxx-extended-api-math-pow2:
+
+Power of Two Utilities
+======================
+
+Defined in the ``<cuda/cmath>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ constexpr
+   bool is_power_of_two(T value) noexcept;
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ constexpr
+   T next_power_of_two(T value) noexcept;
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ constexpr
+   T prev_power_of_two(T value) noexcept;
+
+   } // namespace cuda
+
+The functions provide utilities to determine if an integer value is a power of two, and to compute the next and previous power of two.
+
+**Parameters**
+
+- ``value``: The input value.
+
+**Return value**
+
+- ``is_power_of_two``: Return ``true`` if ``value`` is a power of two, ``false`` otherwise.
+- ``next_power_of_two``: Return the smallest power of two greater than or equal to ``value``.
+- ``prev_power_of_two``: Return the largest power of two less than or equal to ``value``.
+
+**Constraints**
+
+- ``T`` is an integer types. Contrary to ``cuda::std::has_single_bit``, ``cuda::std::bit_floor``, and ``cuda::std::bit_ceil``, ``T`` can be both signed and unsigned.
+
+**Preconditions**
+
+- ``value > 0``
+
+**Performance considerations**
+
+See :ref:`\<cuda/std/bit\> performance considerations <libcudacxx-standard-api-numerics-bit>`
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/cmath>
+    #include <cuda/std/cassert>
+
+    __global__ void pow2_kernel() {
+        assert(!cuda::is_power_of_two(20));
+        assert(cuda::next_power_of_two(20) == 32);
+        assert(cuda::prev_power_of_two(20) == 16);
+    }
+
+    int main() {
+        pow2_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/896Yx3vf8>`__

cccl_upstream/docs/libcudacxx/extended_api/math/round_down.rst

@@ -0,0 +1,62 @@
+.. _libcudacxx-extended-api-math-round-down:
+
+``cuda::round_down``
+====================
+
+Defined in the ``<cuda/cmath>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T, typename U>
+   [[nodiscard]] __host__ __device__ constexpr
+   cuda::std::common_type_t<T, U> round_down(T value, U base_multiple) noexcept;
+
+   } // namespace cuda
+
+The function computes the round down to the largest multiple of an integral or enumerator value :math:`floor(\frac{value}{base\_multiple}) * base\_multiple`
+
+**Parameters**
+
+- ``value``: The value to be rounded down.
+- ``base_multiple``:  The base multiple to which the value rounds down.
+
+**Return value**
+
+``value`` rounded down to the largest multiple of ``base_multiple`` less than or equal to ``value``. If ``value`` is already a multiple of ``base_multiple``, returns ``value``.
+
+**Constraints**
+
+- ``T`` and ``U`` are integer types or enumerators.
+
+**Preconditions**
+
+- ``value >= 0``
+- ``base_multiple > 0``
+
+**Performance considerations**
+
+- The function performs a truncation division followed by a multiplication. It provides better performance than ``(value / base_multiple) * base_multiple`` when the common type is a signed integer
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/cmath>
+    #include <cstdio>
+
+    __global__ void round_up_kernel() {
+        int      value    = 7;
+        unsigned multiple = 3;
+        printf("%d\n", cuda::round_down(value, multiple)); // print "6"
+    }
+
+    int main() {
+        round_up_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/cxGYfMGna>`__

cccl_upstream/docs/libcudacxx/extended_api/math/round_up.rst

@@ -0,0 +1,66 @@
+.. _libcudacxx-extended-api-math-round-up:
+
+``cuda::round_up``
+==================
+
+Defined in the ``<cuda/cmath>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T, typename U>
+   [[nodiscard]] __host__ __device__ constexpr
+   cuda::std::common_type_t<T, U> round_up(T value, U base_multiple) noexcept;
+
+   } // namespace cuda
+
+The function computes the round up to the smallest multiple of an integral or enumerator value :math:`ceil(\frac{value}{base\_multiple}) * base\_multiple`
+
+**Parameters**
+
+- ``value``: The value to be rounded up.
+- ``base_multiple``:  The base multiple to which the value rounds up.
+
+**Return value**
+
+``value`` rounded up to the smallest multiple of ``base_multiple`` greater than or equal to ``value``. If ``value`` is already a multiple of ``base_multiple``, return ``value``.
+
+.. note::
+
+    The result can overflow if ``ceil(value / base_multiple) * base_multiple`` exceeds the maximum value of the common type of ``value`` and ``base_multiple``. The condition is checked in debug mode.
+
+**Constraints**
+
+- ``T`` and ``U`` are integer types or enumerators.
+
+**Preconditions**
+
+- ``value >= 0``
+- ``base_multiple > 0``
+
+**Performance considerations**
+
+- The function performs a ceiling division (``cuda::ceil_div()``) followed by a multiplication
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/cmath>
+    #include <cstdio>
+
+    __global__ void round_up_kernel() {
+        int      value    = 7;
+        unsigned multiple = 3;
+        printf("%d\n", cuda::round_up(value, multiple)); // print "9"
+    }
+
+    int main() {
+        round_up_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/WacYfxsTT>`__

cccl_upstream/docs/libcudacxx/extended_api/math/sincos.rst

@@ -0,0 +1,68 @@
+.. _libcudacxx-extended-api-math-sincos:
+
+``cuda::sincos``
+====================================
+
+Defined in the ``<cuda/cmath>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <class T>
+   struct sincos_result
+   {
+     T sin;
+     T cos;
+   };
+
+   template </*floating-point-type*/ T>
+   [[nodiscard]] __host__ __device__
+   sincos_result<T> sincos(T value) noexcept; // (1)
+
+   template <class Integral>
+   [[nodiscard]] __host__ __device__
+   sincos_result<double> sincos(Integral value) noexcept; // (2)
+
+   } // namespace cuda
+
+Computes :math:`\sin value` and :math:`\cos value` at the same time using more efficient algorithms than if operations were computed separately.
+
+**Parameters**
+
+- ``value``: The input value.
+
+**Return value**
+
+- ``cuda::sincos_result`` object with both values set to ``NaN`` if the input value is :math:`\pm\infty` or ``NaN`` and to results of :math:`\sin value` and :math:`\cos value` otherwise. (1)
+- if ``T`` is an integral type, the input value is treated as ``double``. (2)
+
+**Constraints**
+
+- ``T`` is an arithmetic type.
+
+**Performance considerations**
+
+- If available, the functionality is implemented by compiler builtins, otherwise fallbacks to ``cuda::std::sin(value)`` and ``cuda::std::cos(value)``.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/cmath>
+    #include <cuda/std/cassert>
+
+    __global__ void sincos_kernel() {
+        auto [sin_pi, cos_pi] = cuda::sincos(0.f);
+        assert(sin_pi == 0.f);
+        assert(cos_pi == 1.f);
+    }
+
+    int main() {
+        sincos_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/99PP9s1z6>`__

cccl_upstream/docs/libcudacxx/extended_api/math/uabs.rst

@@ -0,0 +1,56 @@
+.. _libcudacxx-extended-api-math-uabs:
+
+``cuda::uabs``
+====================================
+
+Defined in the ``<cuda/cmath>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ constexpr
+   cuda::std::make_unsigned_t<T> uabs(T value) noexcept;
+
+   } // namespace cuda
+
+The function computes the absolute value of the input value. The result is returned as an unsigned integer type of the same size as the input value. In comparison to the standard ``abs`` function, the ``uabs`` eliminates the undefined behaviour when a signed ``T_MIN`` is passed as an input.
+
+**Parameters**
+
+- ``value``: The input value.
+
+**Return value**
+
+- The unsigned absolute value of the input value.
+
+**Constraints**
+
+- ``T`` is an integer type.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/cmath>
+    #include <cuda/std/cassert>
+    #include <cuda/std/limits>
+
+    __global__ void uabs_kernel() {
+        using cuda::std::numeric_limits;
+
+        assert(cuda::uabs(20) == 20u);
+        assert(cuda::uabs(-32) == 32u);
+        assert(cuda::uabs(numeric_limits<int>::max()) == static_cast<unsigned>(numeric_limits<int>::max()));
+        assert(cuda::uabs(numeric_limits<int>::min()) == static_cast<unsigned>(numeric_limits<int>::max()) + 1);
+    }
+
+    int main() {
+        uabs_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/KEoYfq53G>`__

cccl_upstream/docs/libcudacxx/extended_api/mdspan.rst

@@ -0,0 +1,54 @@
+.. _libcudacxx-extended-api-mdspan:
+
+Mdspan
+======
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   mdspan/host_device_accessor
+   mdspan/layout_stride_relaxed
+   mdspan/restrict_accessor
+   mdspan/shared_memory_accessor
+   mdspan/mdspan_to_dlpack
+   mdspan/dlpack_to_mdspan
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`host/device/managed mdspan and accessor <libcudacxx-extended-api-mdspan-host-device-accessor>`
+     - CUDA memory space ``mdspan`` and accessors
+     - CCCL 3.0.0
+     - CUDA 13.0
+
+   * - :ref:`layout_stride_relaxed <libcudacxx-extended-api-mdspan-layout-stride-relaxed>`
+     - Layout mapping policy with negative/zero strides and offset support
+     - CCCL 3.0.0
+     - CUDA 13.0
+
+   * - :ref:`restrict mdspan and accessor <libcudacxx-extended-api-mdspan-restrict-accessor>`
+     - ``mdspan`` and accessor with the *restrict* aliasing policy
+     - CCCL 3.0.0
+     - CUDA 13.0
+
+   * - :ref:`shared_memory mdspan and accessor <libcudacxx-extended-api-mdspan-shared-memory-accessor>`
+     - ``mdspan`` and accessor for CUDA shared memory
+     - CCCL 3.3.0
+     - CUDA 13.3
+
+   * - :ref:`mdspan to dlpack <libcudacxx-extended-api-mdspan-mdspan-to-dlpack>`
+     - Convert a ``mdspan`` to a ``DLTensor``
+     - CCCL 3.2.0
+     - CUDA 13.2
+
+   * - :ref:`dlpack to mdspan <libcudacxx-extended-api-mdspan-dlpack-to-mdspan>`
+     - Convert a ``DLTensor`` to a ``mdspan``
+     - CCCL 3.2.0
+     - CUDA 13.2

cccl_upstream/docs/libcudacxx/extended_api/mdspan/dlpack_to_mdspan.rst

@@ -0,0 +1,141 @@
+.. _libcudacxx-extended-api-mdspan-dlpack-to-mdspan:
+
+DLPack to ``mdspan``
+====================
+
+This functionality provides a conversion from `DLPack <https://dmlc.github.io/dlpack/latest/>`__ ``DLTensor`` to ``cuda::host_mdspan``, ``cuda::device_mdspan``, and ``cuda::managed_mdspan``.
+
+Defined in the ``<cuda/mdspan>`` header.
+
+Conversion functions
+--------------------
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename ElementType, size_t Rank, typename LayoutPolicy = cuda::layout_stride_relaxed>
+   [[nodiscard]] cuda::host_mdspan<ElementType, cuda::std::dims<Rank, int64_t>, LayoutPolicy>
+   to_host_mdspan(const DLTensor& tensor);
+
+   template <typename ElementType, size_t Rank, typename LayoutPolicy = cuda::layout_stride_relaxed>
+   [[nodiscard]] cuda::device_mdspan<ElementType, cuda::std::dims<Rank, int64_t>, LayoutPolicy>
+   to_device_mdspan(const DLTensor& tensor);
+
+   template <typename ElementType, size_t Rank, typename LayoutPolicy = cuda::layout_stride_relaxed>
+   [[nodiscard]] cuda::managed_mdspan<ElementType, cuda::std::dims<Rank, int64_t>, LayoutPolicy>
+   to_managed_mdspan(const DLTensor& tensor);
+
+   } // namespace cuda
+
+Template parameters
+-------------------
+
+- ``ElementType``: The element type of the resulting ``mdspan``. Must match the ``DLTensor::dtype``.
+- ``Rank``: The number of dimensions. Must match ``DLTensor::ndim``.
+- ``LayoutPolicy``: The layout policy for the resulting ``mdspan``. Defaults to ``cuda::layout_stride_relaxed``. Supported layouts are:
+
+  - ``cuda::std::layout_right`` (C-contiguous, row-major)
+  - ``cuda::std::layout_left`` (Fortran-contiguous, column-major)
+  - ``cuda::std::layout_stride`` (general strided layout)
+  - ``cuda::layout_stride_relaxed`` (general strided layout with negative/zero strides and offset support)
+
+Semantics
+---------
+
+The conversion produces a non-owning ``mdspan`` view of the ``DLTensor`` data:
+
+- For ``layout_right``, ``layout_left``, and ``layout_stride``, the data pointer is computed as ``static_cast<char*>(tensor.data) + tensor.byte_offset``.
+- For ``layout_stride_relaxed``, the data pointer is ``tensor.data`` directly (no ``byte_offset`` adjustment). Instead, ``tensor.byte_offset`` is converted to an element offset (``byte_offset / sizeof(ElementType)``) and stored in the mapping. This ensures that ``mapping(indices...) = offset + sum(index_i * stride_i)`` produces non-negative indices even with negative strides, and ``required_span_size()`` correctly reflects the actual memory span.
+- For ``rank > 0``, ``mdspan.extent(i)`` is ``tensor.shape[i]``.
+- For ``layout_stride`` and ``layout_stride_relaxed``, ``mdspan.stride(i)`` is ``tensor.strides[i]`` (or computed as row-major if ``strides`` is ``nullptr`` for DLPack < v1.2).
+- The device type is validated:
+
+  - ``kDLCPU`` for ``to_host_mdspan``
+  - ``kDLCUDA`` for ``to_device_mdspan``
+  - ``kDLCUDAManaged`` for ``to_managed_mdspan``
+
+Supported element types:
+
+- ``bool``.
+- Signed and unsigned integers.
+- IEEE-754 Floating-point and extended precision floating-point, including ``__half``, ``__nv_bfloat16``, ``__float128``, FP8, FP6, FP4 when available.
+- Complex: ``cuda::std::complex<__half>``, ``cuda::std::complex<float>``, and ``cuda::std::complex<double>``.
+- `CUDA built-in vector types <https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/cpp-language-extensions.html#built-in-types>`__, such as ``int2``, ``float4``, etc.
+- Vector types for extended floating-point, such as ``__half2``, ``__nv_fp8x4_e4m3``, etc.
+
+Constraints
+-----------
+
+- ``LayoutPolicy`` must be one of ``cuda::std::layout_right``, ``cuda::std::layout_left``, ``cuda::std::layout_stride``, or ``cuda::layout_stride_relaxed``.
+- For ``layout_right`` and ``layout_left``, the ``DLTensor`` strides must be compatible with the layout.
+
+Runtime errors
+--------------
+
+The conversion throws ``std::invalid_argument`` in the following cases:
+
+- ``DLTensor::ndim`` does not match the specified ``Rank``.
+- ``DLTensor::dtype`` does not match ``ElementType``.
+- ``DLTensor::data`` is ``nullptr``.
+- ``DLTensor::shape`` is ``nullptr`` (for rank > 0).
+- Any ``DLTensor::shape[i]`` is negative.
+- ``DLTensor::strides`` is ``nullptr`` for DLPack v1.2 or later.
+- ``DLTensor::strides`` is ``nullptr`` for ``layout_left`` with rank > 1 (DLPack < v1.2).
+- ``DLTensor::strides[i]`` is not positive for ``layout_stride``.
+- ``DLTensor::strides`` are not compatible with the requested ``layout_right`` or ``layout_left``.
+- ``DLTensor::byte_offset`` is not a multiple of the element size for ``layout_stride_relaxed``.
+- ``DLTensor::device.device_type`` does not match the target mdspan type.
+- Data pointer is not properly aligned for the element type.
+
+Availability notes
+------------------
+
+- This API is available only when DLPack header is present, namely ``<dlpack/dlpack.h>`` is found in the include path.
+- This API can be disabled by defining ``CCCL_DISABLE_DLPACK`` before including any library headers. In this case, ``<dlpack/dlpack.h>`` will not be included.
+
+References
+----------
+
+- `DLPack C API <https://dmlc.github.io/dlpack/latest/c_api.html>`__ documentation.
+
+Example
+-------
+
+.. code:: cuda
+
+  #include <dlpack/dlpack.h>
+  #include <cuda/mdspan>
+  #include <cuda/std/cassert>
+  #include <cuda/std/cstdint>
+
+  int main() {
+    int data[6] = {0, 1, 2, 3, 4, 5};
+
+    // Create a DLTensor manually for demonstration
+    int64_t shape[2]   = {2, 3};
+    int64_t strides[2] = {3, 1};  // row-major strides
+
+    DLTensor tensor{};
+    tensor.data        = data;
+    tensor.device      = {kDLCPU, 0};
+    tensor.ndim        = 2;
+    tensor.dtype       = DLDataType{kDLInt, 32, 1};
+    tensor.shape       = shape;
+    tensor.strides     = strides;
+    tensor.byte_offset = 0;
+
+    // Convert to host_mdspan
+    auto md = cuda::to_host_mdspan<int, 2>(tensor);
+
+    assert(md.rank() == 2);
+    assert(md.extent(0) == 2 && md.extent(1) == 3);
+    assert(md.stride(0) == 3 && md.stride(1) == 1);
+    assert(md.data_handle() == data);
+    assert(md(0, 0) == 0 && md(1, 2) == 5);
+  }
+
+See also
+--------
+
+- :ref:`libcudacxx-extended-api-mdspan-mdspan-to-dlpack` for the reverse conversion.

cccl_upstream/docs/libcudacxx/extended_api/mdspan/host_device_accessor.rst

@@ -0,0 +1,205 @@
+.. _libcudacxx-extended-api-mdspan-host-device-accessor:
+
+``host/device/managed`` ``mdspan`` and ``accessors``
+====================================================
+
+*Host*, *device*, and *managed* ``mdspan`` allow to express multi-dimensional views of the respective CUDA memory spaces as *vocabulary types* and prevent potential errors.
+
+Types and Traits
+----------------
+
+.. code:: cpp
+
+  template <typename Accessor>
+  using host_accessor;
+
+  template <typename Accessor>
+  using device_accessor;
+
+  template <typename Accessor>
+  using managed_accessor;
+
+Alias types to create accessors tailored for the *host*, *device*, or *managed* memory spaces.
+
+----
+
+.. code:: cpp
+
+  template <typename ElementType,
+            typename Extents,
+            typename LayoutPolicy   = cuda::std::layout_right,
+            typename AccessorPolicy = cuda::std::default_accessor<_ElementType>>
+  using host_mdspan = cuda::std::mdspan<ElementType, Extents, LayoutPolicy, host_accessor<AccessorPolicy>>;
+
+  template <typename ElementType,
+            typename Extents,
+            typename LayoutPolicy   = cuda::std::layout_right,
+            typename AccessorPolicy = cuda::std::default_accessor<_ElementType>>
+  using device_mdspan = cuda::std::mdspan<ElementType, Extents, LayoutPolicy, device_accessor<AccessorPolicy>>;
+
+  template <typename ElementType,
+            typename Extents,
+            typename LayoutPolicy   = cuda::std::layout_right,
+            typename AccessorPolicy = cuda::std::default_accessor<_ElementType>>
+  using managed_mdspan = cuda::std::mdspan<ElementType, Extents, LayoutPolicy, managed_accessor<AccessorPolicy>>;
+
+Alias types to create ``mdspan`` with *host*, *device*, or *managed* accessors.
+
+----
+
+.. code:: cpp
+
+  template <typename T>
+  inline constexpr bool is_host_accessor_v = /* true if T is a host accessor, false otherwise */
+
+  template <typename T>
+  inline constexpr bool is_device_accessor_v = /* true if T is a device accessor, false otherwise */
+
+  template <typename T>
+  inline constexpr bool is_managed_accessor_v = /* true if T is a managed accessor, false otherwise */
+
+  template <typename T>
+  inline constexpr bool is_host_accessible_v = /* true if T is a mdspan/accessor accessible from the host, false otherwise */
+
+  template <typename T>
+  inline constexpr bool is_device_accessible_v = /* true if T is a mdspan/accessor accessible from the device, false otherwise */
+
+----
+
+Features
+--------
+
+**Memory spaces**
+
+*Host*, *device*, and *managed* ``mdspan`` can be created and "sliced" (``cuda::std::submdspan``) on any memory space. However, access to a specific memory space is restricted to the respective *accessor* type.
+
++----------------------------------+------------------+-------------------+
+| ``mdspan`` / memory space access | Host memory      | Device memory     |
++==================================+==================+===================+
+| ``host_mdspan``                  | Allowed          | *Compile error*   |
++----------------------------------+------------------+-------------------+
+| ``device_mdspan``                | *Compile error*  | Allowed           |
++----------------------------------+------------------+-------------------+
+| ``managed_mdspan``               | Allowed *****    | Allowed *****     |
++----------------------------------+------------------+-------------------+
+
+***** the validity of the *managed* memory space is checked at run-time in debug mode (host-side).
+
+**Conversions**
+
++-----------------------------+------------------+-------------------+---------------------+
+|                             | ``host_mdspan``  | ``device_mdspan`` | ``managed_mdspan``  |
++=============================+==================+===================+=====================+
+| ``host_mdspan``             | Allowed          | *Compile error*   | *Compile error*     |
++-----------------------------+------------------+-------------------+---------------------+
+| ``device_mdspan``           | *Compile error*  | Allowed           | *Compile error*     |
++-----------------------------+------------------+-------------------+---------------------+
+| ``managed_mdspan``          | Allowed          | Allowed           | Allowed             |
++-----------------------------+------------------+-------------------+---------------------+
+| Other mdspan                | Allowed          | Allowed           | Allowed             |
++-----------------------------+------------------+-------------------+---------------------+
+
+*Note:* the conversion is ``explicit`` if the base accessor is not directly convertible.
+
+Example 1
+---------
+
+``cuda::host_mdspan`` and ``cuda::device_mdspan`` usage:
+
+.. code:: cpp
+
+    #include <cuda/mdspan>
+
+    using dim = cuda::std::dims<1>;
+
+    __global__ void kernel_d(cuda::device_mdspan<int, dim> md) {
+        md[0] = 0;
+    }
+    __global__ void kernel_h(cuda::host_mdspan<int, dim> md) {
+        // md[0] = 0;  // compile error
+    }
+
+    __host__ void host_function_h(cuda::host_mdspan<int, dim> md) {
+        md[0] = 0;
+    }
+    __host__ void host_function_d(cuda::device_mdspan<int, dim> md) {
+        // md[0] = 0;  // compile error
+    }
+    __host__ void host_function_m(cuda::managed_mdspan<int, dim> md) {
+        md[0] = 0;
+    }
+
+    int main() {
+        int* d_ptr;
+        cudaMalloc(&d_ptr, 4 * sizeof(int));
+        int                 h_ptr[4];
+        cuda::host_mdspan   h_md{h_ptr};
+        cuda::device_mdspan d_md{d_ptr, 4};
+        kernel_d<<<1, 1>>>(d_md);    // ok
+        // kernel_d<<<1, 1>>>(h_md); // compile error
+        host_function_h(h_md);       // ok
+        host_function_d(h_md);       // compile error
+        // host_function_m(h_md);    // compile error
+        cudaFree(d_ptr);
+    }
+
+`See example 1 on Godbolt 🔗 <https://godbolt.org/z/fezxsbjaq>`_
+
+Example 2
+---------
+
+``cuda::managed_mdspan`` usage:
+
+.. code:: cpp
+
+    #include <cuda/mdspan>
+
+    using dim = cuda::std::dims<1>;
+
+    __global__ void kernel_d(cuda::device_mdspan<int, dim> md) {
+        md[0] = 0;
+    }
+
+    __host__ void host_function_h(cuda::host_mdspan<int, dim> md) {
+        md[0] = 0;
+    }
+
+    int main() {
+        int* m_ptr;
+        cudaMallocManaged(&m_ptr, 4 * sizeof(int));
+        cuda::managed_mdspan m_md{m_ptr, 4};
+        kernel_d<<<1, 1>>>(m_md); // ok
+        host_function_h(m_md);    // ok
+
+        cuda::managed_mdspan m_md2{d_ptr, 4};
+        m_md2[0]; // run-time error
+        cudaFree(d_ptr);
+    }
+
+`See example 2 on Godbolt 🔗 <https://godbolt.org/z/Kj39Pe4vP>`_
+
+
+Example 3
+---------
+
+Conversion from other accessors:
+
+.. code:: cpp
+
+    #include <cuda/mdspan>
+
+    using dim = cuda::std::dims<1>;
+
+    int main() {
+        using cuda::std::layout_right;
+        using cuda::std::aligned_accessor;
+        int               h_ptr[4];
+        cuda::std::mdspan md{h_ptr};
+        cuda::host_mdspan h_md = md; // ok
+
+        cuda::std::mdspan<int, dim, layout_right, aligned_accessor<int, 8>> md_a{h_ptr, 4};
+        // cuda::host_mdspan h_md = md_a; // compile-error
+        cuda::host_mdspan    h_md{md_a};  // ok
+    }
+
+`See example 3 on Godbolt 🔗 <https://godbolt.org/z/7dq7vcTWP>`_

cccl_upstream/docs/libcudacxx/extended_api/mdspan/layout_stride_relaxed.rst

@@ -0,0 +1,440 @@
+.. _libcudacxx-extended-api-mdspan-layout-stride-relaxed:
+
+``layout_stride_relaxed``
+=========================
+
+Defined in the ``<cuda/mdspan>`` header.
+
+``layout_stride_relaxed`` is a *LayoutMappingPolicy* which provides a layout mapping where the strides are user-defined and can be negative or zero.
+
+Unlike ``cuda::std::layout_stride``, this layout allows:
+
+- **Negative strides** for reverse iteration.
+- **Zero strides** for broadcasting.
+- **A base offset** to accommodate negative strides.
+- **Compile-time stride values** for static arrays.
+
+.. note::
+
+    This layout is NOT always *unique*, *exhaustive*, or *strided* in the C++ standard sense.
+
+----
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace cuda {
+
+    // Tag value for dynamic stride (analogous to dynamic_extent)
+    inline constexpr ptrdiff_t dynamic_stride = /* implementation-defined */;
+
+    // Strides class template
+    template <class OffsetType, ptrdiff_t... Strides>
+    class strides;
+
+    // Alias for all-dynamic strides
+    template <class OffsetType, size_t Rank>
+    using dstrides = strides<OffsetType, /* Rank dynamic_stride values */>;
+
+    // Alias for steps (synonym for dstrides)
+    template <size_t Rank, class OffsetType = ptrdiff_t>
+    using steps = dstrides<OffsetType, Rank>;
+
+    // Layout policy
+    struct layout_stride_relaxed {
+        template <class Extents,
+                  class Stride = dstrides<make_signed_t<typename Extents::index_type>,
+                                          Extents::rank()>,
+                  class OffsetType = ptrdiff_t>
+        class mapping;
+    };
+
+    } // namespace cuda
+
+----
+
+``strides``
+-----------
+
+Class template to describe the strides of a multi-dimensional array layout. Similar to ``extents``, but for strides. Supports both *static* (compile-time known) and *dynamic* (runtime) stride values.
+
+.. code:: cpp
+
+    // Strides class template
+    template <class OffsetType, ptrdiff_t... Strides>
+    class strides;
+
+    // Alias for all-dynamic strides
+    template <class OffsetType, size_t Rank>
+    using dstrides = strides<OffsetType, /* Rank dynamic_stride values */>;
+
+    // Alias for steps (synonym for dstrides)
+    template <size_t Rank, class OffsetType = ptrdiff_t>
+    using steps = dstrides<OffsetType, Rank>;
+
+**Template Parameters**
+
+- ``OffsetType``: A signed integer type for stride values (supports negative strides) or an *integer-constant-like* type.
+- ``Strides...``: The stride values, where ``dynamic_stride`` indicates a runtime value.
+- ``Rank``: The number of dimensions.
+
+.. list-table::
+    :widths: 40 60
+    :header-rows: 1
+
+    * - **Member Types**
+      - Definition
+    * - ``offset_type``
+      - ``OffsetType``
+    * - ``size_type``
+      - ``cuda::std::make_unsigned_t<offset_type>``
+    * - ``rank_type``
+      - ``cuda::std::size_t``
+
+.. list-table::
+    :widths: 40 60
+    :header-rows: 1
+
+    * - **Static Member Functions**
+      - Description
+    * - ``rank()``
+      - Returns the number of dimensions.
+    * - ``rank_dynamic()``
+      - Returns the number of dynamic strides.
+    * - ``static_stride(rank_type r)``
+      - Returns the static stride at dimension ``r``, or ``dynamic_stride`` if dynamic.
+
+.. list-table::
+    :widths: 40 60
+    :header-rows: 1
+
+    * - **Member Functions**
+      - Description
+    * - ``stride(rank_type r)``
+      - Returns the stride at dimension ``r``.
+
+.. list-table::
+    :widths: 40 60
+    :header-rows: 1
+
+    * - **Non-member Functions**
+      - Description
+    * - ``operator==``
+      - ``true`` if ranks are equal and all extent and stride values compare equal; ``false`` otherwise.
+    * - ``operator!=``
+      - ``false`` if ranks are equal and all extent and stride values compare equal; ``true`` otherwise.
+
+**Constructors**
+
+.. code:: cpp
+
+    // (1) default constructor
+    constexpr strides() noexcept = default;
+
+    // (2) constructor from values
+    template <class... OtherIndexTypes>
+    constexpr explicit strides(OtherIndexTypes... values) noexcept;
+
+    // (3) constructor from span
+    template <class OtherIndexType, size_t Size>
+    constexpr explicit(/*see below*/) strides(cuda::std::span<OtherIndexType, Size> strs) noexcept;
+
+    // (4) constructor from array
+    template <class OtherIndexType, size_t Size>
+    constexpr explicit(/*see below*/) strides(const cuda::std::array<OtherIndexType, Size>& strs) noexcept;
+
+    // (5) constructor from strides
+    template <class OtherIndexType, ptrdiff_t... OtherStrides>
+    constexpr explicit(/*see below*/) strides(const strides<OtherIndexType, OtherStrides...>& other) noexcept;
+
+- **(1)** Default constructor. Value-initializes all dynamic strides to zero.
+- **(2)** Initializes the strides with the provided values.
+
+  - *Constraints*:
+
+    - ``sizeof...(OtherIndexTypes)`` equals ``rank()`` or ``rank_dynamic()``.
+    - ``OtherIndexTypes...`` is convertible and nothrow constructible to ``offset_type``.
+
+  - *Preconditions*:
+
+    - Each value is representable as ``offset_type``.
+
+- **(3)**, **(4)** Initializes the strides from ``span`` or ``array`` values.
+
+  - *Constraints*:
+
+    - ``Size`` equals ``rank_dynamic()`` (implicit) or ``Size`` equals ``rank()`` and ``Size != rank_dynamic()`` (explicit).
+    - ``OtherIndexType`` is convertible and nothrow constructible to ``offset_type``.
+
+  - *Preconditions*:
+
+    - Each value is representable as ``offset_type``.
+
+- **(5)** Initializes the strides from another ``strides`` object.
+
+  - The constructor is ``explicit`` if any static stride in ``Strides...`` corresponds to a ``dynamic_stride`` in ``OtherStrides...``.
+  - *Constraints*:
+
+    - ``sizeof...(OtherStrides)`` equals ``sizeof...(Strides)``.
+    - For each dimension, either stride is ``dynamic_stride``, or both strides are equal.
+
+  - *Preconditions*:
+
+    - Static strides must match their compile-time values.
+    - Each dynamic stride value is representable as ``offset_type``.
+
+``layout_stride_relaxed::mapping``
+----------------------------------
+
+The class template ``layout_stride_relaxed::mapping`` controls how multidimensional indices are mapped with user-defined strides (including negative and zero strides) and an optional offset to a one-dimensional value representing the offset.
+
+.. code:: cpp
+
+    template <class Extents,
+              class Stride = dstrides<make_signed_t<typename Extents::index_type>, Extents::rank()>>
+    class layout_stride_relaxed::mapping;
+
+**Template Parameters**
+
+- ``Extents``: Specifies number of dimensions, their sizes, and which are known at compile time. Must be a specialization of ``cuda::std::extents``.
+- ``Stride``: Specifies the strides for each dimension. Must be a specialization of ``cuda::strides``. Defaults to all-dynamic strides.
+
+**Constraints**
+
+- ``Extents`` must be a specialization of ``cuda::std::extents``.
+- ``Extents::rank()`` must equal ``Stride::rank()``.
+
+.. list-table::
+    :widths: 40 60
+    :header-rows: 1
+
+    * - **Member Types**
+      - Definition
+    * - ``extents_type``
+      - ``Extents``
+    * - ``strides_type``
+      - ``Stride``
+    * - ``index_type``
+      - ``extents_type::index_type``
+    * - ``size_type``
+      - ``extents_type::size_type``
+    * - ``rank_type``
+      - ``extents_type::rank_type``
+    * - ``offset_type``
+      - ``strides_type::offset_type``
+    * - ``layout_type``
+      - ``layout_stride_relaxed``
+
+.. list-table::
+    :widths: 40 60
+    :header-rows: 1
+
+    * - **Static Member Functions**
+      - Description
+    * - ``is_always_unique()``
+      - Returns ``false``. Uniqueness is not guaranteed due to zero/negative strides.
+    * - ``is_always_exhaustive()``
+      - Returns ``false``. Exhaustiveness is not guaranteed due to zero/negative strides.
+    * - ``is_always_strided()``
+      - Returns ``false`` if offset is non-zero (compile-time evaluation). Standard strided behavior is not guaranteed due to offset.
+
+.. list-table::
+    :widths: 40 60
+    :header-rows: 1
+
+    * - **Member Functions**
+      - Description
+    * - ``extents()``
+      - Returns the extents object.
+    * - ``strides()``
+      - Returns the strides object.
+    * - ``offset()``
+      - Returns the base offset (nonnegative).
+    * - ``required_span_size()``
+      - Returns the required span size to cover all valid indices.
+    * - ``operator()(Indices... indices)``
+      - Maps multidimensional indices to a linear index.
+    * - ``is_unique()``
+      - Returns ``false`` (conservative).
+    * - ``is_exhaustive()``
+      - Returns ``false`` (conservative).
+    * - ``is_strided()``
+      - Returns ``true`` if offset is zero, ``false`` otherwise.
+    * - ``stride(rank_type r)``
+      - Returns the stride along dimension ``r``.
+
+.. list-table::
+    :widths: 40 60
+    :header-rows: 1
+
+    * - **Non-member Functions**
+      - Description
+    * - ``operator==``
+      - ``true`` if extents, strides, and offsets are equal; ``false`` otherwise.
+    * - ``operator!=``
+      - ``false`` if extents, strides, and offsets are equal; ``true`` otherwise.
+
+**Constructors**
+
+.. code:: cpp
+
+    // (1) default constructor
+    constexpr mapping() noexcept;
+
+    // (2) copy constructor
+    constexpr mapping(const mapping&) noexcept = default;
+
+    // (3) constructor from strides
+    constexpr mapping(const extents_type& ext,
+                      const strides_type& strides,
+                      offset_type offset = 0) noexcept;
+
+    // (4) converting constructor from mapping
+    template <class OtherMapping>
+    constexpr explicit(/*see below*/) mapping(const OtherMapping& other) noexcept;
+
+- **(1)** Default constructs the mapping, delegating to a ``layout_right`` mapping with default extents.
+- **(2)** Copy constructor.
+- **(3)** Direct-non-list-initializes the extents, strides, and offset with the provided arguments.
+
+  - *Preconditions*:
+
+    - ``offset`` is nonnegative.
+    - ``required_span_size()`` is representable as ``index_type``.
+
+- **(4)** Constructs the mapping by copying extents and strides from ``other``. For ``layout_stride_relaxed`` sources, also copies the offset.
+
+  - The constructor is ``explicit`` if ``OtherMapping::extents_type`` is not convertible to ``extents_type``, or (for non-``layout_stride_relaxed`` mappings) the source is not a ``layout_left``, ``layout_right``, or ``layout_stride`` mapping.
+  - *Constraints*:
+
+    - ``OtherMapping`` is a layout mapping.
+    - ``extents_type`` is constructible from ``OtherMapping::extents_type``.
+
+  - *Preconditions*:
+
+    - ``offset`` is non-negative.
+    - ``required_span_size()`` is representable as ``index_type``.
+
+Examples
+--------
+
+**Compile-time strides (column-major layout)**
+
+.. code:: cpp
+
+    #include <cuda/mdspan>
+    #include <cassert>
+
+    int main() {
+        // 3x4 matrix in column-major order
+        int data[12] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12};
+
+        using extents_t = cuda::std::extents<int, 3, 4>;
+        // Compile-time strides: stride 1 for rows, stride 3 for columns
+        using strides_t = cuda::strides<int, 1, 3>;
+        using mapping_t = cuda::layout_stride_relaxed::mapping<extents_t, strides_t>;
+
+        mapping_t mapping(extents_t{}, strides_t{});
+
+        // Column-major: consecutive elements in same column are adjacent
+        assert(mapping(0, 0) == 0);
+        assert(mapping(1, 0) == 1);
+        assert(mapping(2, 0) == 2);
+        assert(mapping(0, 1) == 3);
+    }
+
+**Negative strides (reverse iteration)**
+
+.. code:: cpp
+
+    #include <cuda/mdspan>
+    #include <cassert>
+
+    int main() {
+        int data[] = {1, 2, 3, 4, 5};
+        // Create a reversed view using negative stride
+        // Offset points to the last element, stride is -1
+        using extents_t = cuda::std::extents<int, 5>;
+        using mapping_t = cuda::layout_stride_relaxed::mapping<extents_t>;
+        using stride_t  = mapping_t::strides_type;
+        mapping_t mapping(extents_t{}, stride_t(-1), 4); // offset=4, stride=-1
+        // Access pattern: mapping(i) = 4 + i * (-1) = 4 - i
+        assert(mapping(0) == 4);
+        assert(mapping(1) == 3);
+        assert(mapping(2) == 2);
+        assert(mapping(3) == 1);
+        assert(mapping(4) == 0);
+    }
+
+**Negative strides (reverse iteration) with constant offset**
+
+.. code:: cpp
+
+    #include <cuda/mdspan>
+    #include <cassert>
+
+    int main() {
+        int data[] = {1, 2, 3, 4, 5};
+        // Create a reversed view using negative stride
+        // Offset points to the last element, stride is -1
+        using extents_t = cuda::std::extents<int, 5>;
+        using offset_t = cuda::std::integral_constant<int, 4>;
+        using strides_t = steps<extents_t::rank()>;
+        using mapping_t = cuda::layout_stride_relaxed::mapping<extents_t, strides_t, offset_t>;
+        mapping_t mapping(extents_t{}, strides_t(-1));
+        assert(mapping(0) == 4);
+        assert(mapping(1) == 3);
+        assert(mapping(2) == 2);
+        assert(mapping(3) == 1);
+        assert(mapping(4) == 0);
+    }
+
+**Zero strides (broadcasting)**
+
+.. code:: cpp
+
+    #include <cuda/mdspan>
+    #include <cassert>
+
+    int main() {
+        int scalar = 42;
+        // Create a broadcast view: single value appears at all indices
+        using extents_t = cuda::std::extents<int, 4, 4>;
+        using mapping_t = cuda::layout_stride_relaxed::mapping<extents_t>;
+        using stride_t  = mapping_t::strides_type;
+
+        mapping_t mapping(extents_t{}, stride_t(0, 0)); // zero strides
+        // All indices map to offset 0
+        assert(mapping(0, 0) == 0);
+        assert(mapping(1, 2) == 0);
+        assert(mapping(3, 3) == 0);
+    }
+
+**Mixed strides**
+
+.. code:: cpp
+
+    #include <cuda/mdspan>
+    #include <cassert>
+
+    int main() {
+        // 2D array with column-major layout but reversed rows
+        // Data: row 0 at end, row 1 in middle, row 2 at start
+        int data[3][4] = {
+            {1, 2,  3,  4},
+            {5, 6,  7,  8},
+            {9, 10, 11, 12}
+        };
+        using extents_t = cuda::std::extents<int, 3, 4>;
+        using mapping_t = cuda::layout_stride_relaxed::mapping<extents_t>;
+        using stride_t  = mapping_t::strides_type;
+        // Reverse rows: stride of -4 in row dimension, +1 in column
+        // Offset to start at last row
+        mapping_t mapping(extents_t{}, stride_t(-4, 1), 8);
+        // Access gives reversed row order
+        assert(mapping(0, 0) == 8); // data[2][0]
+        assert(mapping(1, 0) == 4); // data[1][0]
+        assert(mapping(2, 0) == 0); // data[0][0]
+    }

cccl_upstream/docs/libcudacxx/extended_api/mdspan/mdspan_to_dlpack.rst

@@ -0,0 +1,167 @@
+.. _libcudacxx-extended-api-mdspan-mdspan-to-dlpack:
+
+``mdspan`` to DLPack
+====================
+
+This functionality provides a conversion from ``cuda::host_mdspan``, ``cuda::device_mdspan``, and ``cuda::managed_mdspan`` to `DLPack <https://dmlc.github.io/dlpack/latest/>`__ ``DLTensor`` view.
+
+Defined in the ``<cuda/mdspan>`` header.
+
+Conversion functions
+--------------------
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T, typename Extents, typename Layout, typename Accessor>
+   [[nodiscard]] /*dlpack_tensor*/<Extents::rank()>
+   to_dlpack_tensor(const host_mdspan<T, Extents, Layout, Accessor>& mdspan);
+
+   template <typename T, typename Extents, typename Layout, typename Accessor>
+   [[nodiscard]] /*dlpack_tensor*/<Extents::rank()>
+   to_dlpack_tensor(const device_mdspan<T, Extents, Layout, Accessor>& mdspan);
+
+   template <typename T, typename Extents, typename Layout, typename Accessor>
+   [[nodiscard]] /*dlpack_tensor*/<Extents::rank()>
+   to_dlpack_tensor(const managed_mdspan<T, Extents, Layout, Accessor>& mdspan);
+
+   } // namespace cuda
+
+Types
+-----
+
+``/*dlpack_tensor*/`` is a internal helper class that stores a ``DLTensor`` and owns the backing storage for its ``shape`` and ``strides`` pointers. The class does not use any heap allocation.
+
+.. code:: cuda
+
+  namespace cuda {
+
+  template <size_t Rank>
+  struct /*dlpack_tensor*/ {
+    // cuda::std::array<int64_t, Rank> shape;
+    // cuda::std::array<int64_t, Rank> strides;
+
+    DLTensor get() & const noexcept [[lifetimebound]];
+
+    DLTensor get() && = delete;
+  };
+
+  } // namespace cuda
+
+``/*dlpack_tensor*/`` stores a ``DLTensor`` and owns the backing storage for its ``shape`` and ``strides`` pointers. The class does not use any heap allocation.
+
+.. note:: **Lifetime**
+
+  The ``DLTensor`` associated with ``/*dlpack_tensor*/`` must not outlive the wrapper. If the wrapper is destroyed, the returned ``DLTensor::shape`` and ``DLTensor::strides`` pointers will dangle.
+
+.. note:: **Const-correctness**
+
+  ``DLTensor::data`` points at ``mdspan.data_handle()`` (or is ``nullptr`` if ``mdspan.size() == 0``). If ``T`` is ``const``, the pointer is ``const_cast``'d because ``DLTensor::data`` is unqualified.
+
+Semantics
+---------
+
+The conversion produces a non-owning DLPack view of the ``mdspan`` data and metadata:
+
+- ``DLTensor::ndim`` is ``mdspan.rank()``.
+- For rank > 0, ``DLTensor::shape[i]`` is ``mdspan.extent(i)``.
+- For rank > 0, ``DLTensor::strides[i]`` is ``mdspan.stride(i)``.
+- ``DLTensor::byte_offset`` is always ``0``.
+- ``DLTensor::device`` is:
+
+  - ``{kDLCPU, 0}`` for ``cuda::host_mdspan``
+  - ``{kDLCUDA, /*device_id*/}`` for ``cuda::device_mdspan``
+  - ``{kDLCUDAManaged, 0}`` for ``cuda::managed_mdspan``
+
+Element types are mapped to ``DLDataType`` according to the DLPack conventions, including:
+
+- ``bool``.
+- Signed and unsigned integers.
+- IEEE-754 Floating-point and extended precision floating-point, including ``__half``, ``__nv_bfloat16``, ``__float128``, FP8, FP6, FP4 when available.
+- Complex: ``cuda::std::complex<__half>``, ``cuda::std::complex<float>``, and ``cuda::std::complex<double>``.
+- `CUDA built-in vector types <https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/cpp-language-extensions.html#built-in-types>`__, such as ``int2``, ``float4``, etc.
+- Vector types for extended floating-point, such as ``__half2``, ``__nv_fp8x4_e4m3``, etc.
+
+Constraints
+-----------
+
+- The accessor ``data_handle_type`` must be a pointer type.
+
+Runtime errors
+--------------
+
+- If any ``extent(i)`` or ``stride(i)`` cannot be represented in ``int64_t``, the conversion raises an ``std::invalid_argument`` exception.
+
+Availability notes
+------------------
+
+- This API is available only when DLPack header is present, namely ``<dlpack/dlpack.h>`` is found in the include path.
+- This API can be disabled by defining ``CCCL_DISABLE_DLPACK`` before including any library headers. In this case, ``<dlpack/dlpack.h>`` will not be included.
+
+References
+----------
+
+- `DLPack C API <https://dmlc.github.io/dlpack/latest/c_api.html>`__ documentation.
+
+Example
+-------
+
+.. code:: cuda
+
+  #include <dlpack/dlpack.h>
+  #include <cuda/mdspan>
+  #include <cuda/std/cassert>
+  #include <cuda/std/cstdint>
+
+  int main() {
+    using extents_t = cuda::std::extents<size_t, 2, 3>;
+
+    int data[6] = {0, 1, 2, 3, 4, 5};
+    cuda::host_mdspan<int, extents_t> md{data, extents_t{}};
+
+    auto dl       = cuda::to_dlpack_tensor(md);
+    auto dltensor = dl.get();
+
+    // `dl` owns the shape/stride storage; `dltensor.data` is a non-owning pointer to `data`.
+    assert(dltensor.device.device_type == kDLCPU);
+    assert(dltensor.ndim == 2);
+    assert(dltensor.shape[0] == 2 && dltensor.shape[1] == 3);
+    assert(dltensor.strides[0] == 3 && dltensor.strides[1] == 1);
+    assert(dltensor.data == data);
+  }
+
+Examples of invalid usage:
+
+.. code:: cuda
+
+  #include <dlpack/dlpack.h>
+  #include <cuda/mdspan>
+  #include <cuda/std/cstdint>
+
+  void show_invalid_usage1() {
+    using extents_t = cuda::std::extents<size_t, 2, 3>;
+
+    int data[6] = {0, 1, 2, 3, 4, 5};
+    cuda::host_mdspan<int, extents_t> md{data, extents_t{}};
+
+    // WRONG: calling get() on a temporary is deleted to prevent dangling references.
+    // const DLTensor& dltensor = cuda::to_dlpack_tensor(md).get(); // compile error
+  }
+
+.. code:: cuda
+
+  #include <dlpack/dlpack.h>
+  #include <cuda/mdspan>
+  #include <cuda/std/cstdint>
+
+  int64_t* show_invalid_usage2() {
+    using extents_t = cuda::std::extents<size_t, 2, 3>;
+
+    int data[6] = {0, 1, 2, 3, 4, 5};
+    cuda::host_mdspan<int, extents_t> md{data, extents_t{}};
+
+    auto dl       = cuda::to_dlpack_tensor(md);
+    auto dltensor = dl.get();
+    return dltensor.shape; // WRONG: returns a dangling pointer
+  }

cccl_upstream/docs/libcudacxx/extended_api/mdspan/restrict_accessor.rst

@@ -0,0 +1,80 @@
+.. _libcudacxx-extended-api-mdspan-restrict-accessor:
+
+``restrict`` ``mdspan`` and ``accessor``
+========================================
+
+.. code:: cpp
+
+  template <typename Accessor>
+  using restrict_accessor;
+
+An alias type to create an accessor with the *restrict aliasing policy* starting from an existing accessor.
+
+More information related to the *restrict aliasing policy* can be found in the CUDA programming guide: `__restrict__ keyword <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#restrict>`_.
+
+----
+
+.. code:: cpp
+
+  template <typename ElementType,
+            typename Extents,
+            typename LayoutPolicy   = cuda::std::layout_right,
+            typename AccessorPolicy = cuda::std::default_accessor<_ElementType>>
+  using restrict_mdspan = cuda::std::mdspan<ElementType, Extents, LayoutPolicy, restrict_accessor<AccessorPolicy>>;
+
+An alias type to create an ``mdspan`` with a *restrict aliasing policy* accessor.
+
+----
+
+Traits:
+
+.. code:: cpp
+
+    template <typename T>
+    inline constexpr bool is_restrict_accessor_v = /*true if T is a restrict accessor, false otherwise*/;
+
+    template <typename T>
+    inline constexpr bool is_restrict_mdspan_v = /*true if T is a restrict mdspan, false otherwise*/;
+
+----
+
+**Constraints**:
+
+- Accessor ``data_handle_type`` must be a pointer type.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/mdspan>
+
+    using restrict_mdspan = cuda::restrict_mdspan<int, cuda::std::dims<1>>;
+
+    __host__ __device__ void
+    compute(restrict_mdspan a, restrict_mdspan b, restrict_mdspan c) {
+        c[0] = a[0] * b[0];
+        c[1] = a[0] * b[0];
+        c[2] = a[0] * b[0] * a[1];
+        c[3] = a[0] * a[1];
+        c[4] = a[0] * b[0];
+        c[5] = b[0];
+    }
+
+    int main() {
+        using  dim      = cuda::std::dims<1>;
+        using  mdspan   = cuda::std::mdspan<int, dim>;
+        int    arrayA[] = {1, 2};
+        int    arrayB[] = {5};
+        int    arrayC[] = {9, 10, 11, 12, 13, 14};
+        mdspan mdA{arrayA, dim{1}};
+        mdspan mdB{arrayB, dim{5}};
+        mdspan mdC{arrayC, dim{6}};
+        compute(mdA, mdB, mdC);
+
+        using restrict_aligned_accesor = cuda::std::restrict_accessor<cuda::std::aligned_accessor<int, 8>>;
+        using restrict_aligned_mdspan  = cuda::std::mdspan<int, dim, layout_right, restrict_aligned_accesor>;
+        restrict_aligned_mdspan mdD{mdC};
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/Wjco996z8>`_

cccl_upstream/docs/libcudacxx/extended_api/mdspan/shared_memory_accessor.rst

@@ -0,0 +1,90 @@
+.. _libcudacxx-extended-api-mdspan-shared-memory-accessor:
+
+``shared_memory`` ``mdspan`` and ``accessor``
+=============================================
+
+``shared_memory`` ``mdspan`` and ``accessor`` allow to express multi-dimensional views of the CUDA shared memory space and provide additional safety checks and performance optimizations.
+
+Defined in the ``<cuda/mdspan>`` header.
+
+Types and Traits
+----------------
+
+.. code:: cpp
+
+    namespace cuda {
+
+    template <typename AccessorPolicy>
+    using shared_memory_accessor;
+
+    template <typename ElementType,
+              typename Extents,
+              typename LayoutPolicy   = cuda::std::layout_right,
+              typename AccessorPolicy = cuda::shared_memory_accessor<ElementType>>
+    class shared_memory_mdspan;
+
+    } // namespace cuda
+
+``mdspan`` type and accessor tailored for the *shared* memory space.
+
+----
+
+.. code:: cpp
+
+    namespace cuda {
+
+    template <typename T>
+    inline constexpr bool is_shared_memory_accessor_v = /* true if T is a shared_memory_accessor, false otherwise */;
+
+    template <typename T>
+    inline constexpr bool is_shared_memory_mdspan_v = /* true if T is a shared_memory_mdspan, false otherwise */;
+
+    } // namespace cuda
+
+Features
+--------
+
+**Constraints**
+
+- Accessor ``data_handle_type`` must be a pointer type.
+
+**Preconditions**
+
+- Accessing elements through a ``shared_memory_accessor`` is only allowed in device code.
+- The underlying pointer must be in the *shared* memory space.
+- Access offset must be within the maximum possible shared memory allocation size.
+
+**Performance considerations**
+
+- The functionality guarantees that the accesses use shared memory instructions (``STS/LDS``) rather than generic memory instructions.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/mdspan>
+    #include <cstdio>
+
+    __global__ void kernel() {
+        extern __shared__ int shmem[];
+
+        // Create a shared_memory_mdspan over the dynamic shared memory
+        cuda::shared_memory_mdspan md(shmem, cuda:std::dims<2>{32, 32});
+
+        if (threadIdx.x < 32) {
+             md[threadIdx.x][threadIdx.x] = threadIdx.x; // write on the diagonal
+        }
+        __syncthreads();
+
+        if (threadIdx.x == 0) {
+            printf("md[5][5] = %d\n", md[5][5]); // read from the diagonal
+        }
+    }
+
+    int main() {
+        kernel<<<1, 32, 32 * 32 * sizeof(int)>>>();
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/sojGnKoY9>`_

cccl_upstream/docs/libcudacxx/extended_api/memory.rst

@@ -0,0 +1,84 @@
+.. _libcudacxx-extended-api-memory:
+
+Memory
+======
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   memory/align_down
+   memory/align_up
+   memory/aligned_size
+   memory/discard_memory
+   memory/get_device_address
+   memory/is_address_from
+   memory/is_aligned
+   memory/ptr_rebind
+   memory/ptr_in_range
+   memory/ranges_overlap
+   memory/is_pointer_accessible
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`aligned_size_t <libcudacxx-extended-api-memory-aligned-size>`
+     - Defines an extent of bytes with a statically defined alignment.
+     - libcu++ 1.2.0 / CCCL 2.0.0 (in ``<cuda/memory>`` since CCCL 3.1.0)
+     - CUDA 11.1
+
+   * - :ref:`discard_memory <libcudacxx-extended-api-memory-discard-memory>`
+     - Writes indeterminate values to memory
+     - libcu++ 1.6.0 / CCCL 2.0.0 (in ``<cuda/memory>`` since CCCL 3.1.0)
+     - CUDA 11.5
+
+   * - :ref:`get_device_address <libcudacxx-extended-api-memory-get-device-address>`
+     - Returns a valid address to a device object
+     - CCCL 2.8.0 (in ``<cuda/memory>`` since CCCL 3.1.0)
+     - CUDA 12.9
+
+   * - :ref:`is_address_from and is_object_from <libcudacxx-extended-api-memory-is_address_from>`
+     - Check if a pointer or object is from a specific address space
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`is_aligned <libcudacxx-extended-api-memory-is_aligned>`
+     - Check if a pointer is aligned
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`align_up <libcudacxx-extended-api-memory-align_up>`
+     - Align up a pointer to the specified alignment
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`align_down <libcudacxx-extended-api-memory-align_down>`
+     - Align down a pointer to the specified alignment
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`ptr_rebind <libcudacxx-extended-api-memory-ptr_rebind>`
+     - Rebind a pointer to a different type
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`ptr_in_range <libcudacxx-extended-api-memory-ptr_in_range>`
+     - Check if a pointer is in a range
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`ranges_overlap <libcudacxx-extended-api-memory-ranges_overlap>`
+     - Check if two ranges overlap
+     - CCCL 3.2.0
+     - CUDA 13.2
+
+   * - :ref:`is_host_accessible <libcudacxx-extended-api-memory-is_pointer_accessible>`, :ref:`is_device_accessible <libcudacxx-extended-api-memory-is_pointer_accessible>`, :ref:`is_managed <libcudacxx-extended-api-memory-is_pointer_accessible>`
+     - Check if a pointer is accessible from the host, device, or managed memory
+     - CCCL 3.2.0
+     - CUDA 13.2

cccl_upstream/docs/libcudacxx/extended_api/memory/align_down.rst

@@ -0,0 +1,63 @@
+.. _libcudacxx-extended-api-memory-align_down:
+
+``cuda::align_down``
+====================
+
+Defined in the header ``<cuda/memory>``.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ inline
+   T* align_down(T* ptr, size_t alignment) noexcept;
+
+   } // namespace cuda
+
+The function returns the original pointer or closest pointer smaller than ``ptr`` that is aligned to the specified alignment  :math:`floor\left(\frac{ptr}{alignment}\right) * alignment`.
+
+**Parameters**
+
+- ``ptr``: The pointer.
+- ``alignment``: The alignment.
+
+**Return value**
+
+- The original pointer or closest pointer smaller than ``ptr`` that is aligned to the specified alignment.
+
+**Constraints**
+
+- ``alignment`` must be a power of two.
+- ``alignment >= alignof(T)``.
+- ``ptr`` is aligned to ``alignof(T)``.
+
+**Performance considerations**
+
+- The function is optimized for compile-time values of ``alignment``.
+- The function does not perform any operations if ``alignment == alignof(T)``.
+- The function is translated to a single ``LOP3.LUT`` instruction for other values of ``alignment``.
+- The returned pointer is decorated with ``__builtin_assume_aligned`` to help the compiler generate better code.
+- The returned pointer maintains the same memory space, for example shared memory, as the input pointer.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/memory>
+
+    __global__ void kernel(int* ptr) {
+        auto ptr_align16 = cuda::align_down(ptr, 16);
+        reinterpret_cast<int4*>(ptr_align16)[0] = int4{1, 2, 3, 4};
+    }
+
+    int main() {
+        int* ptr;
+        cudaMalloc(&ptr, 100 * sizeof(int));
+        kernel<<<1, 1>>>(ptr + 20);
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/6acdx3KhY>`__

cccl_upstream/docs/libcudacxx/extended_api/memory/align_up.rst

@@ -0,0 +1,63 @@
+.. _libcudacxx-extended-api-memory-align_up:
+
+``cuda::align_up``
+==================
+
+Defined in the header ``<cuda/memory>``.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ inline
+   T* align_up(T* ptr, size_t alignment) noexcept;
+
+   } // namespace cuda
+
+The function returns the original pointer or closest pointer larger than ``ptr`` that is aligned to the specified alignment :math:`ceil\left(\frac{ptr}{alignment}\right) * alignment`.
+
+**Parameters**
+
+- ``ptr``: The pointer.
+- ``alignment``: The alignment.
+
+**Return value**
+
+- The original pointer or closest pointer larger than ``ptr`` that is aligned to the specified alignment.
+
+**Constraints**
+
+- ``alignment`` must be a power of two.
+- ``alignment >= alignof(T)``.
+- ``ptr`` is aligned to ``alignof(T)``.
+
+**Performance considerations**
+
+- The function is optimized for compile-time values of ``alignment``.
+- The function does not perform any operations if ``alignment == alignof(T)``.
+- The function is translated to ``LOP3.LUT`` + ``IADD.64`` instructions for other values of ``alignment``.
+- The returned pointer is decorated with ``__builtin_assume_aligned`` to help the compiler generate better code.
+- The returned pointer maintains the same memory space, for example shared memory, as the input pointer.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/memory>
+
+    __global__ void kernel(int* ptr) {
+        auto ptr_align16 = cuda::align_up(ptr, 16);
+        reinterpret_cast<int4*>(ptr_align16)[0] = int4{1, 2, 3, 4};
+    }
+
+    int main() {
+        int* ptr;
+        cudaMalloc(&ptr, 100 * sizeof(int));
+        kernel<<<1, 1>>>(ptr);
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/d8e5KETeE>`__

cccl_upstream/docs/libcudacxx/extended_api/memory/aligned_size.rst

@@ -0,0 +1,91 @@
+.. _libcudacxx-extended-api-memory-aligned-size:
+
+``cuda::aligned_size_t``
+========================
+
+Defined in headers ``<cuda/memory>``, ``<cuda/barrier>`` and ``<cuda/pipeline>``:
+
+.. code:: cuda
+
+   template <cuda::std::size_t Alignment>
+   struct cuda::aligned_size_t {
+     static constexpr cuda::std::size_t align = Align;
+     cuda::std::size_t value;
+     __host__ __device__ explicit constexpr aligned_size(cuda::std::size_t size);
+     __host__ __device__ constexpr operator cuda::std::size_t();
+   };
+
+The class template ``cuda::aligned_size_t`` is a *shape* representing an extent of bytes with a statically
+defined (address and size) alignment.
+
+*Preconditions*:
+
+-  The *address* of the extent of bytes must be aligned to an ``Alignment`` alignment boundary.
+-  The *size* of the extent of bytes must be a multiple of the ``Alignment``.
+
+Template Parameters
+-------------------
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``Alignment``
+     - The address and size alignment of the byte extent.
+
+Data Members
+------------
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``align``
+     - The alignment of the byte extent.
+   * - ``value``
+     - The size of the byte extent.
+
+Member Functions
+----------------
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``(constructor)``
+     - Constructs an *aligned size*. If the ``size`` is not a multiple of ``Alignment`` the behavior is undefined.
+   * - ``(destructor)``
+     - Trivial implicit destructor.
+   * - ``operator=``
+     - Trivial implicit copy/move.
+   * - ``operator cuda::std::size_t``
+     - Implicit conversion to `cuda::std::size_t <https://en.cppreference.com/w/cpp/types/size_t>`__.
+
+Notes
+-----
+
+If ``Alignment`` is not a `valid alignment <https://en.cppreference.com/w/c/language/object#Alignment>`_,
+the behavior is undefined.
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/memory>
+
+   __global__ void example_kernel(void* dst, void* src, size_t size) {
+     cuda::barrier<cuda::thread_scope_system> bar;
+     init(&bar, 1);
+
+     // Implementation cannot make assumptions about alignment.
+     cuda::memcpy_async(dst, src, size, bar);
+
+     // Implementation can assume that dst and src are 16-bytes aligned,
+     // and that size is a multiple of 16, and may optimize accordingly.
+     cuda::memcpy_async(dst, src, cuda::aligned_size_t<16>(size), bar);
+
+     bar.arrive_and_wait();
+   }
+
+`See it on Godbolt <https://godbolt.org/z/PWGdfTd7d>`_

cccl_upstream/docs/libcudacxx/extended_api/memory/discard_memory.rst

@@ -0,0 +1,45 @@
+.. _libcudacxx-extended-api-memory-discard-memory:
+
+``cuda::discard_memory``
+========================
+
+Defined in header ``<cuda/memory>``, ``<cuda/discard_memory>`` (deprecated since CCCL 3.2).
+
+.. code:: cuda
+
+    __host__ __device__
+    void discard_memory(volatile void* ptr, size_t nbytes);
+
+Discard modified cache lines without writing back the cached data to memory. The functionality enables using global memory as temporary scratch space. Does **not** generate any HW store operations.
+
+Equivalent to ``memset(ptr, _indeterminate_, nbytes)``.
+
+**Preconditions**
+
+- ``ptr`` points to a valid allocation in *global memory* of size greater or equal to ``nbytes``.
+
+Example
+-------
+
+This kernel needs a scratch pad that does not fit in shared memory, so it uses an allocation in global memory instead:
+
+.. code:: cuda
+
+   #include <cuda/memory>
+
+    __device__ int compute(int* scratch, size_t N);
+
+    __global__ void kernel(const int* in, int* out, int* scratch, size_t N) {
+        // Each thread reads N elements into the scratch pad:
+        for (int i = 0; i < N; ++i) {
+            int idx      = threadIdx.x + i * blockDim.x;
+            scratch[idx] = in[idx];
+        }
+        __syncthreads();
+        // All threads compute on the scratch pad:
+        int result = compute(scratch, N);
+        // All threads discard the scratch pad memory to _hint_ that it does not need to be flushed from the cache:
+        cuda::discard_memory(scratch + threadIdx.x * N, N * sizeof(int));
+        __syncthreads();
+        out[threadIdx.x] = result;
+    }

cccl_upstream/docs/libcudacxx/extended_api/memory/get_device_address.rst

@@ -0,0 +1,72 @@
+.. _libcudacxx-extended-api-memory-get-device-address:
+
+``cuda::get_device_address``
+============================
+
+Defined in the headers ``<cuda/memory>`` and ``<cuda/functional>``.
+
+.. code:: cuda
+
+  namespace cuda {
+
+  template <typename T>
+  [[nodiscard]] __host__ __device__ inline
+  T* get_device_address(T& device_object);                    // (1)
+
+  template <typename T>
+  [[nodiscard]] __host__ inline
+  T* get_device_address(T& device_object, device_ref device); // (2)
+
+  } // namespace cuda
+
+``cuda::get_device_address`` returns a valid pointer to a device object for the current (1) or ``device`` (2) device. It replaces uses of ``cudaGetSymbolAddress``, which requires an inout parameter.
+
+**Parameters**
+
+- ``device_object``: Reference to a device object. (1, 2)
+- ``device``: Device for which the object's address shall be retrieved. (2)
+
+**Constraints**
+
+- ``device_object`` must be a ``__device__`` or ``__constant__`` decorated variable.
+
+Example
+-------
+
+.. code:: cuda
+
+  #include <cuda/devices>
+  #include <cuda/memory>
+
+  __device__ int device_object[] = {42, 1337, -1, 0};
+
+  __global__ void example_kernel(int *data) { ... }
+
+  void example()
+  {
+    cuda::device_ref device{0};
+
+    {
+      T* host_address = cuda::std::addressof(device_object);
+
+      cudaPointerAttributes attributes;
+      cudaError_t status = cudaPointerGetAttributes(&attributes, host_address);
+      assert(status == cudaSuccess);
+      assert(attributes.devicePointer == nullptr);
+
+      // Calling a kernel with host_address would segfault
+      // example_kernel<<<1, 1>>>(host_address);
+    }
+
+    {
+      T* device_address = cuda::get_device_address(device_object, device);
+
+      cudaPointerAttributes attributes;
+      cudaError_t status = cudaPointerGetAttributes(&attributes, device_address);
+      assert(status == cudaSuccess);
+      assert(attributes.devicePointer == device_address);
+
+      // Safe to call a kernel
+      example_kernel<<<1, 1>>>(device_address);
+    }
+  }

cccl_upstream/docs/libcudacxx/extended_api/memory/is_address_from.rst

@@ -0,0 +1,116 @@
+.. _libcudacxx-extended-api-memory-is_address_from:
+
+``cuda::device::is_address_from`` and ``cuda::device::is_object_from``
+======================================================================
+
+Defined in the ``<cuda/memory>`` header.
+
+.. code:: cuda
+
+   namespace cuda::device {
+
+   enum class address_space
+   {
+     global,         // Global state space
+     shared,         // Shared state space
+     constant,       // Constant state space
+     local,          // Local state space
+     grid_constant,  // Kernel function parameter in the parameter state space
+     cluster_shared, // Cluster shared window within the shared state space
+   };
+
+   } // namespace cuda::device
+
+Enumeration of device address spaces used with the ``is_address_from()`` and ``is_object_from()`` functions. See the `PTX ISA documentation for state spaces <https://docs.nvidia.com/cuda/parallel-thread-execution/#state-spaces>`_ for more details.
+
+----
+
+.. code:: cuda
+
+   namespace cuda::device {
+
+   [[nodiscard]] __device__ inline
+   bool is_address_from(const volatile void* ptr, address_space space) noexcept; // (1)
+
+   } // namespace cuda::device
+
+Checks whether a generic-address pointer ``ptr`` is from the specified address space.
+
+----
+
+.. code:: cuda
+
+   namespace cuda::device {
+
+   template <typename T>
+   [[nodiscard]] __device__ inline
+   bool is_object_from(T& obj, address_space space) noexcept; // (2)
+
+   } // namespace cuda::device
+
+Checks whether an object ``obj`` with a generic address is from the specified address space.
+
+----
+
+Unlike the corresponding CUDA intrinsic functions ``__isGlobal()``, ``__isShared()``, ``__isConstant()``, ``__isLocal()``, ``__isGridConstant()``, and ``__isClusterShared()``, ``is_address_from()`` and ``is_object_from()`` are portable across all compute capabilities and, in debug mode, also checks that the pointer is not null.
+
+**Parameters**
+
+- ``ptr``: The pointer. (1)
+- ``obj``: The object. (2)
+- ``space``: The address space. (1, 2)
+
+**Return value**
+
+- Returns ``true`` if the pointer (1) or object (2) is from the specified address space; ``false`` otherwise.
+
+.. note::
+
+  If the GPU architecture does not support the requested address space, the function always returns ``false``.
+
+**Preconditions**
+
+- ``ptr`` must not be null. (1)
+
+**Performance considerations**
+
+- When available, the built-in functions (``__isGlobal()``, ``__isShared()``, ``__isConstant()``, ``__isLocal()``, ``__isGridConstant()``, or ``__isClusterShared()``) are used to determine the address space.
+- If the memory space of the input pointer matches the requested address space,
+  the function marks the pointer as belonging to that address space. For example, a subsequent store to a generic address that maps to shared memory emits an ``STS`` SASS instruction rather than the generic ``ST`` instruction.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/memory>
+
+    __device__   int global_var;
+    __constant__ int constant_var;
+
+    __global__ void kernel(const __grid_constant__ int grid_constant_var)
+    {
+        using cuda::device::address_space;
+        __shared__ int shared_var;
+        int local_var{};
+
+        assert(cuda::device::is_address_from(&global_var, address_space::global));
+        assert(cuda::device::is_address_from(&shared_var, address_space::shared));
+        assert(cuda::device::is_address_from(&constant_var, address_space::constant));
+        assert(cuda::device::is_address_from(&local_var, address_space::local));
+        assert(cuda::device::is_address_from(&grid_constant_var, address_space::grid_constant));
+
+        assert(cuda::device::is_object_from(global_var, address_space::global));
+        assert(cuda::device::is_object_from(shared_var, address_space::shared));
+        assert(cuda::device::is_object_from(constant_var, address_space::constant));
+        assert(cuda::device::is_object_from(local_var, address_space::local));
+        assert(cuda::device::is_object_from(grid_constant_var, address_space::grid_constant));
+    }
+
+    int main(int, char**)
+    {
+        kernel<<<1, 1>>>(42);
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/5ajhe37df>`__

cccl_upstream/docs/libcudacxx/extended_api/memory/is_aligned.rst

@@ -0,0 +1,59 @@
+.. _libcudacxx-extended-api-memory-is_aligned:
+
+``cuda::is_aligned``
+====================
+
+Defined in the header ``<cuda/memory>``.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   [[nodiscard]] __host__ __device__ inline
+   bool is_aligned(const void* ptr, size_t alignment) noexcept
+
+   } // namespace cuda
+
+The function determines if a pointer is aligned to a specific alignment.
+
+**Parameters**
+
+- ``ptr``: The pointer.
+- ``alignment``: The alignment.
+
+**Return value**
+
+- ``true`` if the pointer is aligned to the specified alignment, ``false`` otherwise.
+
+**Constraints**
+
+- ``alignment`` must be a power of two.
+
+.. note::
+
+  The function is similar to the C++ standard library function `cuda::std::is_sufficiently_aligned() <https://en.cppreference.com/w/cpp/memory/is_sufficiently_aligned.html>`__ from the ``<cuda/std/memory>`` header. The differences are the following:
+
+  - ``cuda::is_aligned()`` doesn't have a template parameter and might be less expensive to compile.
+  - ``cuda::is_aligned()`` supports run-time values of ``alignment``.
+  - ``cuda::std::is_sufficiently_aligned()`` additionally checks the compatibility between the alignment of the pointer type and the specified alignment.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/memory>
+
+    __global__ void kernel(const void* ptr) {
+        assert(cuda::is_aligned(ptr, 16));
+    }
+
+    int main() {
+        void* ptr;
+        cudaMalloc(&ptr, 100 * sizeof(int));
+        kernel<<<1, 1>>>(ptr);
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/Tr45EoKsT>`__

cccl_upstream/docs/libcudacxx/extended_api/memory/is_pointer_accessible.rst

@@ -0,0 +1,87 @@
+.. _libcudacxx-extended-api-memory-is_pointer_accessible:
+
+``cuda::is_host_accessible``, ``cuda::is_device_accessible``, ``cuda::is_managed``
+==================================================================================
+
+Defined in the ``<cuda/memory>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   [[nodiscard]] inline
+   bool is_host_accessible(const void* ptr); // (1)
+
+   [[nodiscard]] inline
+   bool is_device_accessible(const void* ptr, device_ref device); // (2)
+
+   [[nodiscard]] inline
+   bool is_managed(const void* ptr); // (3)
+
+   } // namespace cuda
+
+Determines whether the memory referenced by ``ptr`` is accessible from the host (1), from the specified ``device`` (2), or is backed by Unified Memory (managed memory) (3).
+
+- ``is_device_accessible()`` also checks whether the memory is peer-accessible or allocated from a memory pool accessible to the specified ``device``.
+- ``is_host_accessible()`` also checks whether the memory is allocated from a memory pool accessible to the host.
+
+----
+
+**Parameters**
+
+- ``ptr``: A pointer to the memory location to query.
+- ``device``: A ``device_ref`` that denotes the device to query. (2)
+
+**Return value**
+
+- ``true`` if the queried property (host access, device access, or managed allocation) holds; otherwise, ``false``.
+
+.. note::
+
+  A ``__device__`` global array or variable cannot be used directly from host code without first retrieving its address with ``cudaGetSymbolAddress()``.
+
+**Prerequisites**
+
+- The functions are available only when the CUDA Toolkit is available.
+
+**Exceptions**
+
+- The functions throw a ``cuda::cuda_error`` if the underlying driver API calls fail. Note that these functions may also fail with error codes from previously launched asynchronous operations.
+
+**Undefined Behavior**
+
+- The functions have undefined behavior if the pointer is not valid, for example, an already freed pointer.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cassert>
+    #include <cuda/memory>
+    #include <cuda_runtime_api.h>
+
+    int main() {
+        cuda::device_ref dev{0};
+        void* host_ptr    = nullptr;
+        void* device_ptr  = nullptr;
+        void* managed_ptr = nullptr;
+
+        cudaMallocHost(&host_ptr, 1024);
+        cudaMalloc(&device_ptr, 1024);
+        cudaMallocManaged(&managed_ptr, 1024);
+
+        assert(cuda::is_host_accessible(host_ptr));
+        assert(!cuda::is_device_accessible(host_ptr, dev));
+
+        assert(cuda::is_device_accessible(device_ptr, dev));
+        assert(!cuda::is_host_accessible(device_ptr));
+
+        assert(cuda::is_host_accessible(managed_ptr));
+        assert(cuda::is_device_accessible(managed_ptr, dev));
+        assert(cuda::is_managed(managed_ptr));
+
+        cudaFreeHost(host_ptr);
+        cudaFree(device_ptr);
+        cudaFree(managed_ptr);
+    }

cccl_upstream/docs/libcudacxx/extended_api/memory/ptr_in_range.rst

@@ -0,0 +1,67 @@
+.. _libcudacxx-extended-api-memory-ptr_in_range:
+
+``cuda::ptr_in_range``
+======================
+
+Defined in the header ``<cuda/memory>``.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ constexpr
+   bool ptr_in_range(T* ptr, T* start, T* end) noexcept;
+
+   } // namespace cuda
+
+Checks whether ``ptr`` lies inside the half-open interval ``[start, end)``.
+
+**Template parameters**
+
+- ``T``: The type of the pointer.
+
+**Parameters**
+
+- ``ptr``: The pointer being tested.
+- ``start``: Pointer to the first element in the range.
+- ``end``: Pointer to one past the last element in the range.
+
+**Return value**
+
+- ``true`` when the pointer lies in ``[start, end)``, ``false`` otherwise.
+
+**Preconditions**
+
+- ``end`` must be greater than or equal to ``start``.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/memory>
+
+    __global__ void kernel(float* data, size_t count) {
+        float* first = data;
+        float* last  = data + count;
+
+        float* elem_ptr = data + threadIdx.x;
+        if (cuda::ptr_in_range(elem_ptr, first, last)) {
+            *elem_ptr = static_cast<float>(threadIdx.x);
+        }
+    }
+
+    int main() {
+        size_t N          = 32;
+        float* device_ptr = nullptr;
+        cudaMalloc(&device_ptr, N * sizeof(float));
+
+        kernel<<<1, N>>>(device_ptr, N);
+        cudaDeviceSynchronize();
+
+        cudaFree(device_ptr);
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/sMz76hGEc>`__

cccl_upstream/docs/libcudacxx/extended_api/memory/ptr_rebind.rst

@@ -0,0 +1,70 @@
+.. _libcudacxx-extended-api-memory-ptr_rebind:
+
+``cuda::ptr_rebind``
+====================
+
+Defined in the header ``<cuda/memory>``.
+
+.. code:: cuda
+
+    namespace cuda {
+
+    template <typename U, typename T>
+    [[nodiscard]] __host__ __device__
+    U* ptr_rebind(T* ptr) noexcept;
+
+    template <typename U, typename T>
+    [[nodiscard]] __host__ __device__
+    const U* ptr_rebind(const T* ptr) noexcept;
+
+    template <typename U, typename T>
+    [[nodiscard]] __host__ __device__
+    volatile U* ptr_rebind(volatile T* ptr) noexcept;
+
+    template <typename U, typename T>
+    [[nodiscard]] __host__ __device__
+    const volatile U* ptr_rebind(const volatile T* ptr) noexcept;
+
+    } // namespace cuda
+
+The functions return the pointer ``ptr`` cast to type ``U*`` or ``const U*``. They are shorter and safer alternative to ``reinterpret_cast``.
+
+**Parameters**
+
+- ``ptr``: The pointer.
+
+**Return value**
+
+- The pointer cast to type ``U*`` or ``const U*``.
+
+**Constraints**
+
+- ``ptr`` must be aligned to ``alignof(U)`` and ``alignof(T)``.
+
+**Performance considerations**
+
+- The returned pointer is decorated with ``__builtin_assume_aligned`` with the ``alignof(U)`` value to help the compiler generate better code.
+- The returned pointer maintains the same memory space, for example shared memory, as the input pointer.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/memory>
+    #include <cuda/std/cstdint>
+
+    __global__ void kernel(const int* ptr, volatile int* ptr2) {
+        auto ptr_res1 = cuda::ptr_rebind<uint64_t>(ptr);  // ptr_res1: const uint64_t*
+        auto ptr_res2 = cuda::ptr_rebind<uint64_t>(ptr2); // ptr_res2: volatile uint64_t*
+    }
+
+    int main() {
+        int* ptr;
+        cudaMalloc(&ptr, 100 * sizeof(int));
+        kernel<<<1, 1>>>(ptr);
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/bavzabce9>`__

cccl_upstream/docs/libcudacxx/extended_api/memory/ranges_overlap.rst

@@ -0,0 +1,60 @@
+.. _libcudacxx-extended-api-memory-ranges_overlap:
+
+``cuda::ranges_overlap``
+========================
+
+Defined in the ``<cuda/memory>`` header.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template <typename T>
+   [[nodiscard]] __host__ __device__ constexpr
+   bool ranges_overlap(T lhs_start, T lhs_end, T rhs_start, T rhs_end) noexcept;
+
+   } // namespace cuda
+
+Returns ``true`` when the half-open byte ranges ``[lhs_start, lhs_end)`` and ``[rhs_start, rhs_end)`` intersect.
+
+**Constraints**
+
+- ``T`` must be a forward iterator.
+
+**Parameters**
+
+- ``lhs_start``: The beginning of the first range.
+- ``lhs_end``: The end of the first range.
+- ``rhs_start``: The beginning of the second range.
+- ``rhs_end``: The end of the second range.
+
+**Return value**
+
+- ``true`` when the two ranges overlap, ``false`` otherwise.
+
+**Performance considerations**
+
+- The function is optimized when the ranges are contiguous and random access iterators.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/memory>
+    #include <cuda/std/cassert>
+
+    __global__ void overlap_kernel() {
+        int arrayA[10];
+        int arrayB[10];
+        assert(cuda::ranges_overlap(arrayA + 2, arrayA + 7, arrayA, arrayA + 10)); // overlap
+        assert(!cuda::ranges_overlap(arrayA, arrayA + 10, arrayB, arrayB + 10));   // no overlap
+    }
+
+    int main() {
+        overlap_kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+        return 0;
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/nasnWz9Tv>`__

cccl_upstream/docs/libcudacxx/extended_api/memory_access_properties.rst

@@ -0,0 +1,41 @@
+.. _libcudacxx-extended-api-memory-access-properties:
+
+Memory access properties
+------------------------
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   memory_access_properties/access_property
+   memory_access_properties/annotated_ptr
+   memory_access_properties/apply_access_property
+   memory_access_properties/associate_access_property
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`cuda::access_property <libcudacxx-extended-api-memory-access-properties-access-property>`
+     - Represents a memory access property
+     - libcu++ 1.6.0 / CCCL 2.0.0 /
+     - CUDA 11.5
+
+   * - :ref:`cuda::annotated_ptr <libcudacxx-extended-api-memory-access-properties-annotated-ptr>`
+     - Binds an access property to a pointer
+     - libcu++ 1.6.0 / CCCL 2.0.0
+     - CUDA 11.5
+   * - :ref:`cuda::apply_access_property <libcudacxx-extended-api-memory-access-properties-apply-access-property>`
+     - Applies access property to memory
+     - libcu++ 1.6.0 / CCCL 2.0.0
+     - CUDA 11.5
+
+   * - :ref:`cuda::associate_access_property <libcudacxx-extended-api-memory-access-properties-associate-access-property>`
+     - Associates access property with raw pointer
+     - libcu++ 1.6.0 / CCCL 2.0.0
+     - CUDA 11.5

cccl_upstream/docs/libcudacxx/extended_api/memory_access_properties/access_property.rst

@@ -0,0 +1,221 @@
+.. _libcudacxx-extended-api-memory-access-properties-access-property:
+
+``cuda::access_property``
+=========================
+
+Defined in header ``<cuda/annotated_ptr>``.
+
+The class ``cuda::access_property`` provides an opaque encoding for *L2 cache memory residence* control and *memory space* properties. It is used in combination with :ref:`cuda::annotated_ptr <libcudacxx-extended-api-memory-access-properties-annotated-ptr>`, :ref:`cuda::associate_access_property <libcudacxx-extended-api-memory-access-properties-associate-access-property>` and :ref:`cuda::apply_access_property <libcudacxx-extended-api-memory-access-properties-apply-access-property>` to *request* the application of properties to memory operations.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   class access_property {
+     public:
+       // Static memory space property:
+       struct shared {};
+       struct global {};
+
+       // Static global memory residence control property:
+       struct normal {
+           __host__ __device__ constexpr operator cudaAccessProperty() const noexcept;
+       };
+       struct persisting {
+           __host__ __device__ constexpr operator cudaAccessProperty() const noexcept;
+       };
+       struct streaming {
+           __host__ __device__ constexpr operator cudaAccessProperty() const noexcept;
+       };
+
+       access_property() noexcept = default;
+
+       // Constructors from static global memory residence control properties:
+       __host__ __device__ constexpr access_property(global)     noexcept;
+       __host__ __device__ constexpr access_property(normal)     noexcept;
+       __host__ __device__ constexpr access_property(streaming)  noexcept;
+       __host__ __device__ constexpr access_property(persisting) noexcept;
+
+       // Dynamic interleaved global memory residence control property constructors:
+       __host__ __device__ constexpr access_property(normal,     float probability) noexcept;
+       __host__ __device__ constexpr access_property(streaming,  float probability) noexcept;
+       __host__ __device__ constexpr access_property(persisting, float probability) noexcept;
+       __host__ __device__ constexpr access_property(normal,     float probability, streaming) noexcept;
+       __host__ __device__ constexpr access_property(persisting, float probability, streaming) noexcept;
+
+       // Dynamic range global memory residence control property constructors:
+       __host__ __device__ access_property(void* ptr, size_t primary_bytes, size_t total_bytes, normal) noexcept;
+       __host__ __device__ access_property(void* ptr, size_t primary_bytes, size_t total_bytes, streaming) noexcept;
+       __host__ __device__ access_property(void* ptr, size_t primary_bytes, size_t total_bytes, persisting) noexcept;
+       __host__ __device__ access_property(void* ptr, size_t primary_bytes, size_t total_bytes, global,     streaming) noexcept;
+       __host__ __device__ access_property(void* ptr, size_t primary_bytes, size_t total_bytes, normal,     streaming) noexcept;
+       __host__ __device__ access_property(void* ptr, size_t primary_bytes, size_t total_bytes, persisting, streaming) noexcept;
+       __host__ __device__ access_property(void* ptr, size_t primary_bytes, size_t total_bytes, streaming,  streaming) noexcept;
+   };
+
+   } // namespace cuda
+
+Kinds of Access Properties
+--------------------------
+
+Access properties are either *static* compile-time values or *dynamic* runtime values. The following properties
+of a memory access are provided:
+
+*Shared Memory property*:
+
+   .. _libcudacxx-extended-api-memory-access-properties-access-property-shared:
+
+- ``cuda::access_property::shared``: memory access to the shared memory space.
+
+*Global Memory properties*:
+
+   .. _libcudacxx-extended-api-memory-access-properties-access-property-global:
+
+- ``cuda::access_property::global``: memory access to the global memory space *without* indicating an expected frequency of access to that memory, namely the access behavior is not modified.
+
+   .. _libcudacxx-extended-api-memory-access-properties-access-property-normal:
+
+- ``cuda::access_property::normal``: memory access to the global memory space expecting the memory to be accessed as frequent as other memory.
+
+   .. _libcudacxx-extended-api-memory-access-properties-access-property-persisting:
+
+- ``cuda::access_property::persisting``: memory access to the global memory space expecting the memory to be accessed more frequently than other memory; this priority is suitable for data that should remain persistent in cache.
+
+   .. _libcudacxx-extended-api-memory-access-properties-access-property-streaming:
+
+- ``cuda::access_property::streaming``: memory access to the global memory space expecting the memory to be accessed infrequently; this priority is suitable for streaming data.
+
+**Note**: The difference between ``cuda::access_property::global`` and ``cuda::access_property::normal`` is subtle.
+The ``cuda::access_property::normal`` hints that the pointer points to the global address space *and* the memory will
+be accessed with "normal frequency", while ``cuda::access_property::global`` only hints that the pointer points to
+the global address-space, it does not hint about how frequent the accesses will be.
+
+.. warning::
+
+   The behavior of *requesting* the application of ``cuda::access_property`` to memory accesses, or their association
+   with memory addresses, outside of the corresponding address space is *undefined*
+   (note: even if that address is not used). The correctness of the input pointer and memory properties are verified in debug mode.
+
+----
+
+Global Memory Property Definition
+---------------------------------
+
+The L2 residence control can be specified in two ways:
+
+- **Interleaved**: A memory address is accessed with a property with a given ``probability``, while the remaining ``1 - probability`` accesses are performed with a second one.
+- **Range**: The first ``primary_bytes`` of a memory address is accessed with one property and the remaining ``total_bytes - primary_bytes`` addresses with a second one.
+
+Default constructor
+-------------------
+
+.. code:: cuda
+
+   access_property() noexcept = default;
+
+**Effects**: as if ``access_property(global)`` (unchanged).
+
+Static global memory residence control property constructors
+------------------------------------------------------------
+
+.. code:: cuda
+
+   __host__ __device__ constexpr access_property::access_property(global) noexcept;
+   __host__ __device__ constexpr access_property::access_property(normal) noexcept;
+   __host__ __device__ constexpr access_property::access_property(streaming) noexcept;
+   __host__ __device__ constexpr access_property::access_property(persisting) noexcept;
+
+**Effects**: as if ``access_property(PROPERTY, 1.0)`` where ``PROPERTY`` is one of ``global`` (unchanged), ``normal``, ``streaming``, or ``persisting``.
+
+Dynamic interleaved global memory residence control property constructors
+-------------------------------------------------------------------------
+
+.. code:: cuda
+
+   __host__ __device__ constexpr access_property::access_property(normal,     float probability) noexcept;
+   __host__ __device__ constexpr access_property::access_property(streaming,  float probability) noexcept;
+   __host__ __device__ constexpr access_property::access_property(persisting, float probability) noexcept;
+   __host__ __device__ constexpr access_property::access_property(normal,     float probability, streaming) noexcept;
+   __host__ __device__ constexpr access_property::access_property(persisting, float probability, streaming) noexcept;
+
+**Preconditions**: ``0 < probability <= 1.0``.
+
+**Effects**: constructs an *interleaved* access property that *requests* the first and third arguments - access properties - to be applied with ``probability`` and ``1 - probability`` to memory accesses. The overloads without a third argument request applying ``global`` (unchanged) with ``1 - probability``.
+
+Dynamic range global memory residence control property constructors
+-------------------------------------------------------------------
+
+.. code:: cuda
+
+   __host__ __device__ access_property::access_property(void* ptr, size_t leading_bytes, size_t total_bytes, normal) noexcept;
+   __host__ __device__ access_property::access_property(void* ptr, size_t leading_bytes, size_t total_bytes, streaming) noexcept;
+   __host__ __device__ access_property::access_property(void* ptr, size_t leading_bytes, size_t total_bytes, persisting) noexcept;
+   __host__ __device__ access_property::access_property(void* ptr, size_t leading_bytes, size_t total_bytes, normal,     streaming) noexcept;
+   __host__ __device__ access_property::access_property(void* ptr, size_t leading_bytes, size_t total_bytes, persisting, streaming) noexcept;
+
+..
+
+   note: pointer arithmetic below performed ``char* ptr`` instead of
+   ``void* ptr``
+
+**Preconditions**:
+
+   - ``ptr`` is a generic pointer that is *valid* to cast to a pointer to the global memory address space.
+   - ``0 < leading_bytes <= total_bytes <= 4GB``.
+
+**Postconditions**: memory accesses requesting the application of this property must be in range ``[ptr, ptr + total_bytes)``.
+
+**Effects**: the fourth and fifth arguments, access properties, are called *primary* and *secondary* properties. The overloads without a fifth argument use ``global`` as the *secondary* property. Constructs a *range* access property *requesting* the properties to be **approximately** applied to memory accesses as follows:
+
+-  *primary property* to accesses in address-range:   ``[ptr, ptr + leading_bytes)``
+-  *secondary property* to accesses in address-range: ``[ptr + leading_bytes, ptr + total_bytes)``
+
+**Note**: This property enables two main use cases:
+
+1. Unary range ``[ptr, ptr + total_bytes)`` with *primary property* by using ``leading_bytes == total_bytes``.
+
+2. Binary range ``[ptr, ptr + leading_bytes)`` and ``[ptr + leading_bytes, ptr + total_bytes)`` with *primary* and
+   *secondary properties* respectively.
+
+Conversion operators
+--------------------
+
+.. code:: cuda
+
+   __host__ __device__ constexpr access_property::normal::operator     cudaAccessProperty() const noexcept;
+   __host__ __device__ constexpr access_property::streaming::operator  cudaAccessProperty() const noexcept;
+   __host__ __device__ constexpr access_property::persisting::operator cudaAccessProperty() const noexcept;
+
+Allows ``constexpr cuda::access_property::normal{}``, ``cuda::access_property::streaming{}``, and ``cuda::access_property::persisting{}`` to be used in lieu of the corresponding CUDA Runtime `cudaAccessProperty <https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html#group__CUDART__TYPES_1g4991a8bc9c2356a8da28d093a1da6758>`_. See also `L2 Policy for Persisting Accesses <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#l2-policy-for-persisting-accesses>`_.
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/access_property>
+
+   __global__ void kernel(int* global_ptr, size_t num_bytes) {
+       __shared__ int smem;
+       cuda::access_property shared_prop{&smem, cuda::access_property::shared{}};
+       cuda::access_property streaming_prop{global_ptr, sizeof(int), sizeof(int), cuda::access_property::streaming{}};
+       cuda::access_property streaming_interleaved_prop{cuda::access_property::streaming{}, 1.0};
+       cuda::access_property persisting_prop{global_ptr, num_bytes, num_bytes, cuda::access_property::persisting{});
+   }
+
+   __global__ void undefined_behavior(int* global_ptr) { // verified in debug mode
+       __shared__ int smem;
+       // Associating pointers with mismatching address spaces is undefined:
+       cuda::access_property{global_ptr, cuda::access_property::shared{}}; // undefined behavior
+       cuda::access_property{&smem, cuda::access_property::normal{}};      // undefined behavior
+       cuda::access_property{&smem, cuda::access_property::streaming{}};   // undefined behavior
+       cuda::access_property{&smem, cuda::access_property::persisting{}};  // undefined behavior
+
+       // Using a zero probability or probability out-of-range (0, 1] is undefined:
+       cuda::access_property{cuda::access_property::streaming{}, 0.0f};    // undefined behavior
+       cuda::access_property{cuda::access_property::streaming{}, 2.0f};    // undefined behavior
+
+       // Providing size values out-of-range is undefined:
+       cuda::access_property{global_ptr, 0, 0, cuda::access_property::streaming{}, 0.0f}; // undefined behavior
+       cuda::access_property{global_ptr, 8, 4, cuda::access_property::streaming{}, 2.0f}; // undefined behavior
+   }

cccl_upstream/docs/libcudacxx/extended_api/memory_access_properties/annotated_ptr.rst

@@ -0,0 +1,341 @@
+.. _libcudacxx-extended-api-memory-access-properties-annotated-ptr:
+
+``cuda::annotated_ptr``
+=======================
+
+Defined in header ``<cuda/annotated_ptr>``.
+
+``cuda::annotated_ptr`` is a pointer annotated with an access property that *may* be applied to its memory operations.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   template<typename Type, typename Property>
+   class annotated_ptr {
+   public:
+     using value_type      = Type;
+     using size_type       = size_t;
+     using reference       = value_type&;
+     using pointer         = value_type*;
+     using const_pointer   = const value_type*;
+     using difference_type = ptrdiff_t;
+
+     annotated_ptr() noexcept = default;
+
+     __host__ __device__ explicit constexpr annotated_ptr(pointer) noexcept;
+
+     template <typename RuntimeProperty>
+     __host__ __device__ annotated_ptr(pointer, RuntimeProperty) noexcept;
+
+     template <typename T, typename P>
+     __host__ __device__ annotated_ptr(const annotated_ptr<T,P>&) noexcept;
+
+     __host__ __device__ constexpr explicit operator bool() const noexcept;
+
+     [[nodiscard]] __host__ __device__ pointer   get() const noexcept;
+     [[nodiscard]] __host__ __device__ reference operator*() const noexcept;
+     [[nodiscard]] __host__ __device__ pointer   operator->() const noexcept;
+     [[nodiscard]] __host__ __device__ reference operator[](ptrdiff_t) const noexcept;
+     [[nodiscard]] __host__ __device__ constexpr difference_type operator-(annotated_ptr) const noexcept;
+
+   private:
+     pointer  ptr;  // exposition only
+     Property prop; // exposition only
+   };
+
+   } // namespace cuda
+
+.. note::
+  If ``Property`` is :ref:`cuda::access_property <libcudacxx-extended-api-memory-access-properties-access-property>`,
+  namely a dynamic property with a runtime value,
+  then ``sizeof(cuda::annotated_ptr<Type, cuda::access_property>) == 2 * sizeof(Type*)``. Otherwise, its size is ``sizeof(Type*)``.
+
+In contrast to :ref:`cuda::associate_access_property <libcudacxx-extended-api-memory-access-properties-associate-access-property>`, ``cuda::annotated_ptr`` maintains the association between the pointer and the property when passed across translation units.
+
+**Constraints**
+
+``Property`` is one of:
+
+-  :ref:`cuda::access_property::shared <libcudacxx-extended-api-memory-access-properties-access-property-shared>`,
+-  :ref:`cuda::access_property::global <libcudacxx-extended-api-memory-access-properties-access-property-global>`,
+-  :ref:`cuda::access_property::persisting <libcudacxx-extended-api-memory-access-properties-access-property-persisting>`,
+-  :ref:`cuda::access_property::normal <libcudacxx-extended-api-memory-access-properties-access-property-normal>`,
+-  :ref:`cuda::access_property::streaming <libcudacxx-extended-api-memory-access-properties-access-property-streaming>` or,
+-  :ref:`cuda::access_property <libcudacxx-extended-api-memory-access-properties-access-property>`:
+   a type-erased specification that allows ``annotated_ptr`` to set the access property at runtime value.
+
+**Semantics**
+
+.. list-table::
+   :widths: 25 30 40
+   :header-rows: 1
+
+   * - Pointer Expression
+     - ``cuda::annotated_ptr<T, P>``
+     - Description
+
+   * - ``T* a``
+     - ``cuda::annotated_ptr<T, P> a``
+     - Non-``const`` pointer to non-``const`` memory
+
+   * - ``T const * a``
+     - ``cuda::annotated_ptr<T const, P> a``
+     - Non-``const`` pointer to ``const`` memory
+
+   * - ``T* const a``
+     - ``const cuda::annotated_ptr<T, P> a``
+     - ``const`` pointer to non-``const`` memory
+
+   * - ``T const* const a``
+     - ``const cuda::annotated_ptr<T const, P> a``
+     - ``const`` pointer to ``const`` memory
+
+   * - ``val = *a;``
+     - ``val = *a;``
+     - Dereference operator to load an element
+
+   * - ``*a = val;``
+     - ``*a = val;``
+     - Dereference operator to store an element
+
+   * - ``val = a[n];``
+     - ``val = a[n];``
+     - Subscript operator to load an element
+
+   * - ``a[n] = val;``
+     - ``a[n] = val;``
+     - Subscript operator to store an element
+
+   * - ``T* a = nullptr;``
+     - ``annotated_ptr<T, P> a = nullptr;``
+     - ``nullptr`` initialization
+
+   * - ``n = a - b;``
+     - ``n = a - b;``
+     - Difference operator
+
+   * - ``if (a) { ... }``
+     - ``if (a) { ... }``
+     - Bool conversion
+
+*Note*: It is not a drop-in replacement for pointers since, among others, it does not:
+
+-  model any `Iterator <https://en.cppreference.com/w/cpp/iterator>`_ concept,
+-  implement `cuda::std::pointer_traits <https://en.cppreference.com/w/cpp/memory/pointer_traits>`_,    `cuda::std::iterator_traits <https://en.cppreference.com/w/cpp/iterator/iterator_traits>`_, etc.
+-  have the same variance as pointer.
+
+----
+
+Constructors and Assignment
+---------------------------
+
+Default constructor
+~~~~~~~~~~~~~~~~~~~
+
+.. code:: cuda
+
+   annotated_ptr() noexcept = default;
+
+**Effects**:  as if constructed by ``annotated_ptr(nullptr)``;
+
+Constructor from pointer
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: cuda
+
+   constexpr explicit annotated_ptr(pointer ptr);
+
+Constructs an ``annotated_ptr`` requesting associating ``ptr`` with ``Property``.
+
+**Constraints**:
+
+- If ``Property`` is :ref:`cuda::access_property::shared <libcudacxx-extended-api-memory-access-properties-access-property-shared>`, :ref:`cuda::access_property::global <libcudacxx-extended-api-memory-access-properties-access-property-global>`,  :ref:`cuda::access_property::normal <libcudacxx-extended-api-memory-access-properties-access-property-normal>`, :ref:`cuda::access_property::streaming <libcudacxx-extended-api-memory-access-properties-access-property-streaming>`, :ref:`cuda::access_property::persisting <libcudacxx-extended-api-memory-access-properties-access-property-persisting>`, or `cuda::access_property` (dynamic).
+
+**Preconditions**:
+
+- If ``Property`` is :ref:`cuda::access_property::shared <libcudacxx-extended-api-memory-access-properties-access-property-shared>`, then ``ptr`` must be a generic pointer that is a valid pointer to the *shared memory* address space.
+- If ``Property`` is  not :ref:`cuda::access_property::shared <libcudacxx-extended-api-memory-access-properties-access-property-shared>`, then ``ptr`` must be a generic pointer    that is a valid pointer to the *global memory* address space.
+
+Constructor from pointer and access property
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code:: cuda
+
+   template <typename RuntimeProperty>
+   annotated_ptr(pointer ptr, RuntimeProperty prop);
+
+Constructs an ``annotated_ptr`` requesting the association of ``ptr`` with the property ``prop``.
+
+**Constraints**:
+
+-  ``RuntimeProperty`` is any of :ref:`cuda::access_property::global <libcudacxx-extended-api-memory-access-properties-access-property-global>`,
+   :ref:`cuda::access_property::normal <libcudacxx-extended-api-memory-access-properties-access-property-normal>`,
+   :ref:`cuda::access_property::streaming <libcudacxx-extended-api-memory-access-properties-access-property-streaming>`,
+   :ref:`cuda::access_property::persisting <libcudacxx-extended-api-memory-access-properties-access-property-persisting>`, or
+   :ref:`cuda::access_property <libcudacxx-extended-api-memory-access-properties-access-property>` (same as *global*).
+
+**Preconditions**:
+
+- ``ptr`` is a pointer to a valid allocation in the *global memory* address space.
+
+Copy Constructor from a different ``annotated_ptr``
+----------------------------------------------------
+
+.. code:: cuda
+
+   template <typename T, typename P>
+   constexpr annotated_ptr(const annotated_ptr<T, P>& a);
+
+Constructs an ``annotated_ptr`` for the same pointer as the input ``annotated_ptr``.
+
+**Constraints**
+
+-  ``annotated_ptr<Type, Property>::pointer`` is assignable from ``annotated_ptr<T, P>::pointer``.
+-  ``Property`` is either ``cuda::access_property`` (*dynamic*) or ``P``.
+-  ``Property`` and ``P`` specify the same memory space.
+
+**Preconditions**
+
+- ``pointer`` is compatible with ``Property``.
+
+Explicit conversion operator to ``bool``
+----------------------------------------
+
+.. code:: cuda
+
+   constexpr operator bool() const noexcept;
+
+**Returns**: ``false`` if the pointer is a ``nullptr``, ``true`` otherwise.
+
+Raw pointer access
+------------------
+
+.. code:: cuda
+
+   pointer get() const noexcept;
+
+**Returns**: A pointer derived from the ``annotated_ptr``.
+
+Operators
+---------
+
+Dereference
+~~~~~~~~~~~
+
+.. code:: cuda
+
+   reference operator*() const noexcept;
+
+**Returns**: value pointed by ``annotated_ptr``.
+
+**Preconditions**
+
+The underlying pointer is not null.
+
+Pointer-to-member
+~~~~~~~~~~~~~~~~~
+
+.. code:: cuda
+
+   pointer operator->() const noexcept;
+
+**Preconditions**
+
+- The underlying pointer is not null.
+
+**Returns**: underlying pointer.
+
+Subscript
+~~~~~~~~~
+
+.. code:: cuda
+
+   reference operator[](ptrdiff_t i) const noexcept;
+
+**Returns**: reference to element ``i``.
+
+**Preconditions**
+
+- The underlying pointer plus the offset ``i`` is not null.
+
+Pointer distance
+~~~~~~~~~~~~~~~~
+
+.. code:: cuda
+
+   constexpr difference_type operator-(annotated_ptr p) const;
+
+**Returns**: Difference of pointers, as-if ``get() - p.get()``.
+
+**Preconditions**
+
+- ``ptr >= p``.
+
+----
+
+Example
+-------
+
+Given three input and output vectors ``x``, ``y``, and ``z``, and two arrays of coefficients ``a`` and ``b``, all of length ``N``:
+
+.. code:: cuda
+
+    size_t N;
+    int* x, *y, *z;
+    int* a, *b;
+
+the grid-strided kernel:
+
+.. code:: cuda
+
+    __global__ void update(const int* x, const int* a, const int* b, size_t N) {
+        auto g = cooperative_groups::this_grid();
+        for (int i = g.thread_rank(); idx < N; idx += g.size()) {
+            x[i] = a[i] * x[i] + b[i];
+        }
+    }
+
+updates ``x``, ``y``, and ``z`` as follows:
+
+.. code:: cuda
+
+   update<<<grid, block>>>(x, a, b, N);
+   update<<<grid, block>>>(y, a, b, N);
+   update<<<grid, block>>>(z, a, b, N);
+
+The elements of ``a`` and ``b`` are used in all kernels. If ``N`` is large enough, elements of ``a`` and ``b`` might be evicted from the L2 cache, requiring these to be re-loaded from memory in the next ``update``.
+
+We can make the ``update`` kernel generic to allow the caller to pass ``cuda::annotated_ptr`` objects that hint at how memory will be accessed:
+
+.. code:: cuda
+
+    template <typename PointerX, typename PointerA, typename PointerB>
+    __global__ void update_template(PointerX x, PointerA a, PointerB b, size_t N) {
+        auto g = cooperative_groups::this_grid();
+        for (int idx = g.thread_rank(); idx < N; idx += g.size()) {
+            x[idx] = a[idx] * x[idx] + b[idx];
+        }
+    }
+
+With ``cuda::annotated_ptr``, the caller can then specify the temporal locality of the memory accesses:
+
+.. code:: cuda
+
+   // Frequent accesses to "a" and "b"; infrequent accesses to "x" and "y":
+   cuda::annotated_ptr<const int, cuda::access_property::persisting> a_persistent{a}, b_persistent{b};
+   cuda::annotated_ptr<int, cuda::access_property::streaming>        x_streaming{x}, y_streaming{y};
+   update_template<<<grid, block>>>(x_streaming, a_persistent, b_persistent, N);
+   update_template<<<grid, block>>>(y_streaming, a_persistent, b_persistent, N);
+
+   // Infrequent accesses to "a" and "b"; frequent accesses to "z":
+   cuda::annotated_ptr<const int, cuda::access_property::streaming> a_streaming{a}, b_streaming{b};
+   cuda::annotated_ptr<int, cuda::access_property::persisting>      z_persistent{z};
+   update_template<<<grid, block>>>(z_persistent, a_streaming, b_streaming, N);
+
+   // Different kernel, "update_z", uses "z" again one last time.
+   // Since "z" was accessed as "persisting" by the previous kernel,
+   // parts of it are more likely to have previously survived in the L2 cache.
+   update_z<<<grid, block>>>(z, ...);
+
+Notice how the raw pointers to ``a`` and ``b`` can be wrapped by both ``annotated_ptr<T, persistent>`` and ``annotated_ptr<T, streaming>``, and accesses through each pointer applies the corresponding access property.

cccl_upstream/docs/libcudacxx/extended_api/memory_access_properties/apply_access_property.rst

@@ -0,0 +1,101 @@
+.. _libcudacxx-extended-api-memory-access-properties-apply-access-property:
+
+``cuda::apply_access_property``
+===============================
+
+Defined in header ``<cuda/annotated_ptr>``.
+
+.. code:: cuda
+
+   template <typename ShapeT>
+   [[nodiscard]] __host__ __device__
+   void apply_access_property(const volatile void* ptr, ShapeT shape, cuda::access_property::persisting) noexcept;
+
+   template <typename ShapeT>
+   [[nodiscard]] __host__ __device__
+   void apply_access_property(const volatile void* ptr, ShapeT shape, cuda::access_property::normal) noexcept;
+
+Prefetch memory in the L2 cache starting at ``ptr`` applying a residence control property.
+
+**Constraints**
+
+- ``ShapeT`` is either ``size_t`` or :ref:`cuda::aligned_size_t <libcudacxx-extended-api-memory-aligned-size>`.
+- Two properties are supported:
+
+    -  :ref:`cuda::access_property::persisting <libcudacxx-extended-api-memory-access-properties-access-property-persisting>`
+    -  :ref:`cuda::access_property::normal <libcudacxx-extended-api-memory-access-properties-access-property-normal>`
+
+**Preconditions**
+
+- ``ptr`` points to a valid allocation for ``shape`` in the *global memory* address space.
+
+    -  if ``ShapeT`` is ``aligned_size_t<N>(sz)``, then ``ptr`` is aligned to an ``N``-bytes alignment boundary, and
+    -  for all offsets ``i`` in the extent of ``shape``, namely ``i`` in ``[0, shape)``, then the expression ``*(ptr + i)`` does not exhibit undefined behavior.
+
+*Note*:  currently ``apply_access_property`` is ignored on the host.
+
+Example
+-------
+
+Given three input and output vectors ``x``, ``y``, and ``z``, and two arrays of coefficients ``a`` and ``b``, all of length ``N``:
+
+.. code:: cuda
+
+   size_t N;
+   int* x, *y, *z;
+   int* a, *b;
+
+the grid-strided kernel:
+
+.. code:: cuda
+
+    __global__ void update(const int* x, const int* a, const int* b, size_t N) {
+        auto g = cooperative_groups::this_grid();
+        for (int idx = g.thread_rank(); idx < N; idx += g.size()) {
+            x[idx] = a[idx] * x[idx] + b[idx];
+        }
+    }
+
+updates ``x``, ``y``, and ``z`` as follows:
+
+.. code:: cuda
+
+    update<<<grid, block>>>(x, a, b, N);
+    update<<<grid, block>>>(y, a, b, N);
+    update<<<grid, block>>>(z, a, b, N);
+
+The elements of ``a`` and ``b`` are used in all kernels. For certain values of ``N``, this may prevent parts of ``a`` and ``b`` from being evicted from the L2 cache, avoiding reloading these from memory in the subsequent ``update`` kernel.
+
+With :ref:`cuda::access_property <libcudacxx-extended-api-memory-access-properties-access-property>` and :ref:`cuda::apply_access_property <libcudacxx-extended-api-memory-access-properties-apply-access-property>`, we can write kernels that specify that ``a`` and ``b`` are accessed more often in the ``pin`` kernel and with normal access in the ``unpin`` kernel:
+
+.. code:: cuda
+
+    __global__ void pin(int* a, int* b, size_t N) {
+        auto g = cooperative_groups::this_grid();
+        for (int idx = g.thread_rank(); idx < N; idx += g.size()) {
+            cuda::apply_access_property(a + idx, sizeof(int), cuda::access_property::persisting{});
+            cuda::apply_access_property(b + idx, sizeof(int), cuda::access_property::persisting{});
+        }
+    }
+
+    __global__ void unpin(int* a, int* b, size_t N) {
+        auto g = cooperative_groups::this_grid();
+        for (int idx = g.thread_rank(); idx < N; idx += g.size()) {
+            cuda::apply_access_property(a + idx, sizeof(int), cuda::access_property::normal{});
+            cuda::apply_access_property(b + idx, sizeof(int), cuda::access_property::normal{});
+        }
+    }
+
+which we can launch before and after the ``update`` kernels:
+
+.. code:: cuda
+
+   pin<<<grid, block>>>(a, b, N);
+   update<<<grid, block>>>(x, a, b, N);
+   update<<<grid, block>>>(y, a, b, N);
+   update<<<grid, block>>>(z, a, b, N);
+   unpin<<<grid, block>>>(a, b, N);
+
+This does not require modifying the ``update`` kernel, and for certain values of ``N`` prevents ``a`` and ``b`` from having to be re-loaded from memory.
+
+The ``pin`` and ``unpin`` kernels can be fused into the kernels for the ``x`` and ``z`` updates by modifying these kernels.

cccl_upstream/docs/libcudacxx/extended_api/memory_access_properties/associate_access_property.rst

@@ -0,0 +1,49 @@
+.. _libcudacxx-extended-api-memory-access-properties-associate-access-property:
+
+``cuda::associate_access_property``
+===================================
+
+Defined in header ``<cuda/annotated_ptr>``.
+
+.. code:: cuda
+
+   template <typename T, typename Property>
+   [[nodiscard]] __host__ __device__
+   T* associate_access_property(T* ptr, Property prop) noexcept;
+
+Associate an :ref:`cuda::access_property <libcudacxx-extended-api-memory-access-properties-access-property>` to the input pointer, such that subsequent memory operations with the returned pointer *or* pointers derived from it *may* apply the access property.
+
+-  The "association" is *not* part of the value representation of the pointer.
+-  The compiler is allowed to drop the association; it does not have a functional consequence.
+-  The association *may* hold through simple expressions, sequence of simple statements, or fully inlined function
+   calls where the pointer value or C++ reference is provably unchanged; this includes offset pointers used for
+   array access.
+-  The association is *not* expected to hold through the ABI of an unknown function call, e.g., when the pointer is
+   passed through a separately-compiled function interface, unless link-time optimizations are used.
+
+**Constraints**
+
+- ``Property`` is convertible to :ref:`cuda::access_property <libcudacxx-extended-api-memory-access-properties-access-property>`.
+
+**Preconditions**:
+
+- If ``Property`` is :ref:`cuda::access_property::shared <libcudacxx-extended-api-memory-access-properties-access-property-shared>`, then it must be valid to cast the generic pointer ``ptr`` to a pointer to the *shared memory* address space.
+
+- If ``Property`` is one of :ref:`cuda::access_property::global <libcudacxx-extended-api-memory-access-properties-access-property-global>`, :ref:`cuda::access_property::persisting <libcudacxx-extended-api-memory-access-properties-access-property-persisting>`, :ref:`cuda::access_property::normal <libcudacxx-extended-api-memory-access-properties-access-property-normal>`, or     :ref:`cuda::access_property::streaming <libcudacxx-extended-api-memory-access-properties-access-property-streaming>`, then it must be valid to cast the generic pointer ``ptr`` to a pointer to the *global memory* address space.
+
+*Note*: currently ``associate_access_property`` is ignored by nvcc and nvc++ on the host.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/cooperative_groups.h>
+
+    __global__ void memcpy_kernel(const int* in, int* out) {
+        __shared__ int smem[N];
+        auto in1          = cuda::associate_access_property(in, cuda::access_property::streaming{});
+        auto idx          = cooperative_groups::this_grid().thread_rank();
+        smem[threadIdx.x] = in1[idx]; // streaming access
+        // compute...
+    }

cccl_upstream/docs/libcudacxx/extended_api/memory_model.rst

@@ -0,0 +1,222 @@
+.. _libcudacxx-extended-api-memory-model:
+
+Memory model
+============
+
+Standard C++ presents a view that the cost to synchronize threads is uniform and low.
+
+CUDA C++ is different: the cost to synchronize threads grows as threads are further apart. It is low across threads
+within a block, but high across arbitrary threads in the system running on multiple GPUs and CPUs.
+
+To account for non-uniform thread synchronization costs that are not always low, CUDA C++ extends the standard C++
+memory model and concurrency facilities in the ``cuda::`` namespace with **thread scopes**, retaining the syntax and
+semantics of standard C++ by default.
+
+.. _libcudacxx-extended-api-memory-model-thread-scopes:
+
+Thread Scopes
+-------------
+
+A **thread scope** specifies the kind of threads that can synchronize with each other using a synchronization primitive such
+as :ref:`atomic <libcudacxx-extended-api-synchronization-atomic>` or
+:ref:`barrier <libcudacxx-extended-api-synchronization-barrier>`.
+
+.. code:: cuda
+
+   namespace cuda {
+
+   enum thread_scope {
+     thread_scope_system,
+     thread_scope_device,
+     thread_scope_block,
+     thread_scope_thread
+   };
+
+   }  // namespace cuda
+
+Scope Relationships
+~~~~~~~~~~~~~~~~~~~
+
+Each program thread is related to each other program thread by one or more thread scope relations:
+   - Each thread in the system is related to each other thread in the system by the *system* thread scope:
+     ``thread_scope_system``.
+   - Each GPU thread is related to each other GPU thread in the same CUDA device and within the same `memory
+     synchronization domain <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#memory-synchronization-domains>`__
+     by the *device* thread scope: ``thread_scope_device``.
+   - Each GPU thread is related to each other GPU thread in the same CUDA thread block by the *block* thread scope:
+     ``thread_scope_block``.
+   - Each thread is related to itself by the ``thread`` thread scope: ``thread_scope_thread``.
+
+Synchronization primitives
+--------------------------
+
+Types in namespaces ``std::`` and ``cuda::std::`` have the same behavior as corresponding types in namespace ``cuda::``
+when instantiated with a scope of ``cuda::thread_scope_system``.
+
+Atomicity
+---------
+
+An atomic operation is atomic at the scope it specifies if:
+
+   - it specifies a scope other than ``thread_scope_system``, **or**
+   - the scope is ``thread_scope_system`` and:
+
+      -  it affects an object in `system allocated memory <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#um-unified-memory-programming-hd>`__ and `pageableMemoryAccess <https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html#group__CUDART__TYPES_1gg49e2f8c2c0bd6fe264f2fc970912e5cddc80992427a92713e699953a6d249d6f>`__ is ``1`` [0],  **or**
+      -  it affects an object in `managed
+         memory <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#um-unified-memory-programming-hd>`__
+         and
+         `concurrentManagedAccess <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaDeviceProp.html#structcudaDeviceProp_116f9619ccc85e93bc456b8c69c80e78b>`__
+         is ``1``, **or**
+      -  it affects an object in `mapped
+         memory <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#mapped-memory>`__ and
+         `hostNativeAtomicSupported <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaDeviceProp.html#structcudaDeviceProp_1ef82fd7d1d0413c7d6f33287e5b6306f>`__
+         is ``1``, **or**
+      -  it is a load or store that affects a naturally-aligned object of
+         sizes ``1``, ``2``, ``4``, ``8``, or ``16`` bytes on `mapped
+         memory <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#mapped-memory>`__ [1],
+         **or**
+      -  it affects an object in GPU memory, only GPU threads access it, and
+          - `*val` returned from `cudaDeviceGetP2PAttribute(&val, cudaDevP2PAttrNativeAtomicSupported, srcDev, dstDev) <https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html#group__CUDART__TYPES_1g2f597e2acceab33f60bd61c41fea0c1b>`__ between each accessing `srcDev` and the GPU where the object resides, `dstDev`, is ``1``, or
+          - only GPU threads from a single GPU concurrently access it.
+
+.. note::
+   - [0] If `PageableMemoryAccessUsesHostPagetables <https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html#group__CUDART__TYPES_1gg49e2f8c2c0bd6fe264f2fc970912e5cdc228cf8983c97d0e035da72a71494eaa>`__ is ``0`` then atomic operations to memory mapped file or ``hugetlbfs`` allocations are not atomic.
+   - [1] If `hostNativeAtomicSupported <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaDeviceProp.html#structcudaDeviceProp_1ef82fd7d1d0413c7d6f33287e5b6306f>`__ is ``0``, atomic load or store operations at system scope that affect a
+     naturally-aligned 16-byte wide object in
+     `unified memory <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#um-unified-memory-programming-hd>`__ or
+     `mapped memory <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#mapped-memory>`__ require system
+     support. NVIDIA is not aware of any system that lacks this support and there is no CUDA API query available to
+     detect such systems.
+
+Refer to the `CUDA programming guide <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html>`__
+for more information on
+`system allocated memory <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#um-unified-memory-programming-hd>`__,
+`managed memory <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#um-unified-memory-programming-hd>`__,
+`mapped memory <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#mapped-memory>`__,
+CPU memory, and GPU memory.
+
+Data Races
+----------
+
+Modify `intro.races paragraph 21 <https://eel.is/c++draft/intro.races#21>`__ of ISO/IEC IS 14882 (the C++ Standard)
+as follows:
+
+   The execution of a program contains a data race if it contains two potentially concurrent conflicting actions, at
+   least one of which is not atomic **at a scope that includes the thread that performed the other operation**, and neither
+   happens before the other, except for the special case for signal handlers described below.
+   Any such data race results in undefined behavior. […]
+
+Modify `thread.barrier.class paragraph 4 <https://eel.is/c++draft/thread.barrier.class#4>`__ of ISO/IEC IS
+14882 (the C++ Standard) as follows
+
+   4. Concurrent invocations of the member functions of ``barrier``, other than its destructor, do not introduce data
+   races **as if they were atomic operations**. […]
+
+Modify `thread.latch.class paragraph 2 <https://eel.is/c++draft/thread.latch.class#2>`__ of ISO/IEC IS 14882
+(the C++ Standard) as follows:
+
+   2. Concurrent invocations of the member functions of ``latch``, other than its destructor, do not introduce data
+   races **as if they were atomic operations**. […]
+
+Modify `thread.sema.cnt paragraph 3 <https://eel.is/c++draft/thread.sema.cnt#3>`__ of ISO/IEC IS 14882
+(the C++ Standard) as follows:
+
+   3. Concurrent invocations of the member functions of ``counting_semaphore``, other than its destructor, do not
+   introduce data races **as if they were atomic operations**.
+
+Modify `thread.stoptoken.intro paragraph 5 <https://eel.is/c++draft/thread#stoptoken.intro-5>`__ of ISO/IEC IS
+14882 (the C++ Standard) as follows:
+
+   Calls to the functions ``request_stop``, ``stop_requested``, and ``stop_possible`` do not introduce data
+   races **as if they were atomic operations**. […]
+
+Modify `atomics.fences paragraph 2 through 4 <https://eel.is/c++draft/atomics.fences#2>`__ of ISO/IEC IS 14882 (the
+C++ Standard) as follows:
+
+   A release fence A synchronizes with an acquire fence B if there exist atomic operations X and Y, both operating on
+   some atomic object M, such that A is sequenced before X, X modifies M, Y is sequenced before B, and Y reads the value
+   written by X or a value written by any side effect in the hypothetical release sequence X would head if it were a
+   release operation, **and each operation (A, B, X, and Y) specifies a scope that includes the thread that performed
+   each other operation**.
+
+   A release fence A synchronizes with an atomic operation B that performs an acquire operation on an atomic object M if
+   there exists an atomic operation X such that A is sequenced before X, X modifies M, and B reads the value written by
+   X or a value written by any side effect in the hypothetical release sequence X would head if it were a release
+   operation, **and each operation (A, B, and X) specifies a scope that includes the thread that performed each other
+   operation**.
+
+   An atomic operation A that is a release operation on an atomic object M synchronizes with an acquire fence B if
+   there exists some atomic operation X on M such that X is sequenced before B and reads the value written by A or a
+   value written by any side effect in the release sequence headed by A, **and each operation (A, B, and X) specifies
+   a scope that includes the thread that performed each other operation**.
+
+.. _libcudacxx-extended-api-memory-model-message-passing:
+
+Example: Message Passing
+------------------------
+
+The following example passes a message stored to the ``x`` variable by a
+thread in block ``0`` to a thread in block ``1`` via the flag ``f``:
+
+.. code:: cpp
+
+   int x = 0;
+   int f = 0;
+
+.. list-table::
+   :widths: 50 50
+   :header-rows: 1
+
+   * - Thread 0 Block 0
+     - Thread 0 Block 1
+   * -
+       .. code:: cpp
+
+          x = 42;
+          cuda::atomic_ref<int, cuda::thread_scope_device> flag(f);
+          flag.store(1, memory_order_release);
+     -
+       .. code:: cpp
+
+          cuda::atomic_ref<int, cuda::thread_scope_device> flag(f);
+          while(flag.load(memory_order_acquire) != 1);
+          assert(x == 42);
+
+In the following variation of the previous example, two threads
+concurrently access the ``f`` object without synchronization, which
+leads to a **data race**, and exhibits **undefined behavior**:
+
+.. code:: cpp
+
+   int x = 0;
+   int f = 0;
+
+.. list-table::
+   :widths: 50 50
+   :header-rows: 1
+
+   * - Thread 0 Block 0
+     - Thread 0 Block 1
+   * -
+       .. code:: cpp
+
+          x = 42;
+          cuda::atomic_ref<int, cuda::thread_scope_block> flag(f);
+          flag.store(1, memory_order_release); // UB: data race
+     -
+       .. code:: cpp
+
+          cuda::atomic_ref<int, cuda::thread_scope_device> flag(f);
+          while(flag.load(memory_order_acquire) != 1); // UB: data race
+          assert(x == 42);
+
+While the memory operations on ``f`` - the store and the loads - are
+atomic, the scope of the store operation is "block scope". Since the
+store is performed by Thread 0 of Block 0, it only includes all other
+threads of Block 0. However, the thread doing the loads is in Block 1,
+i.e., it is not in a scope included by the store operation performed in
+Block 0, causing the store and the load to not be "atomic", and
+introducing a data-race.
+
+For more examples see the `PTX memory consistency model litmus
+tests <https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#axioms>`__.

cccl_upstream/docs/libcudacxx/extended_api/memory_resource.rst

@@ -0,0 +1,69 @@
+.. _libcudacxx-extended-api-memory-resources:
+.. _libcudacxx-memory-resource-async:
+
+Memory Resources
+================
+
+.. toctree::
+   :maxdepth: 1
+
+   memory_resource/properties
+   Resources <memory_resource/resource>
+   Type-erased wrappers <memory_resource/wrappers>
+   Resource utilities <memory_resource/resource_utilities>
+
+The ``<cuda/memory_resource>`` header provides a standard C++ interface for *heterogeneous*, *stream-ordered* memory
+allocation tailored to the needs of CUDA C++ developers. This design builds off of the success of the `RAPIDS Memory Manager (RMM) <https://github.com/rapidsai/rmm>`__
+project and evolves the design based on lessons learned.
+
+``<cuda/memory_resource>`` is not intended to replace RMM, but instead moves the definition of the memory allocation
+interface to a more centralized home in CCCL. RMM will remain as a collection of implementations of the ``cuda::mr``
+interfaces.
+
+At a high level, the header provides:
+
+.. list-table::
+   :widths: 30 50 20
+   :header-rows: 0
+
+   * - :ref:`cuda::get_property <libcudacxx-extended-api-memory-resources-properties>`
+     - Infrastructure to tag a user defined type with a given property
+     - stable CCCL 3.1.0 / CUDA 13.1, experimental CCCL 2.2.0 / CUDA 12.3
+   * - :ref:`cuda::mr::{synchronous_}resource <libcudacxx-extended-api-memory-resources-resource>` and
+       :ref:`cuda::mr::{synchronous_}resource_with <libcudacxx-extended-api-memory-resources-resource>`
+     - Concepts that provide proper constraints for arbitrary memory resources.
+     - stable CCCL 3.1.0 / CUDA 13.1, experimental CCCL 2.2.0 / CUDA 12.3
+   * - :ref:`cuda::mr::{synchronous_}resource_ref <libcudacxx-extended-api-memory-resources-resource-ref>`
+     - A non-owning type-erased memory resource wrapper that enables consumers to specify properties of resources that they expect.
+     - stable CCCL 3.2.0 / CUDA 13.2, experimental CCCL 2.2.0 / CUDA 12.3
+   * - :ref:`cuda::mr::any_resource <libcudacxx-extended-api-memory-resources-any-resource>` and
+       :ref:`cuda::mr::any_synchronous_resource <libcudacxx-extended-api-memory-resources-any-synchronous-resource>`
+     - Owning type-erased wrappers for stream-ordered and synchronous resources.
+     - stable CCCL 3.2.0 / CUDA 13.2, experimental CCCL 2.2.0 / CUDA 12.3
+   * - :ref:`try_get_property <libcudacxx-extended-api-memory-resources-try-get-property>`
+     - Property query helper for type-erased resource wrappers.
+     - stable CCCL 3.2.0 / CUDA 13.2
+   * - :ref:`cuda::mr::resource_cast <libcudacxx-extended-api-memory-resources-resource-cast>` and
+       :ref:`cuda::mr::dynamic_resource_cast <libcudacxx-extended-api-memory-resources-dynamic-resource-cast>`
+     - Cast helpers for type-erased resource wrappers.
+     - stable CCCL 3.3.0 / CUDA 13.3
+   * - :ref:`cuda::mr::shared_resource <libcudacxx-extended-api-memory-resources-shared-resource>`
+     - Reference-counted wrapper to share a resource instance across objects.
+     - stable CCCL 3.2.0 / CUDA 13.2, experimental CCCL 2.2.0 / CUDA 12.3
+   * - :ref:`cuda::mr::synchronous_resource_adapter <libcudacxx-extended-api-memory-resources-synchronous-adapter>`
+     - Adapter that enables synchronous resources to work with streams.
+     - stable CCCL 3.2.0 / CUDA 13.2, experimental CCCL 2.2.0 / CUDA 12.3
+   * - :ref:`cuda::mr::get_memory_resource <libcudacxx-extended-api-memory-resources-get-memory-resource>`
+     - Customization point for retrieving a memory resource from an object or execution environment.
+     - stable CCCL 3.2.0 / CUDA 13.2
+
+These features are an evolution of `std::pmr::memory_resource <https://en.cppreference.com/w/cpp/header/memory_resource>`__
+that was introduced in C++17. While ``std::pmr::memory_resource`` provides a polymorphic memory resource that can be
+adopted through inheritance, it is not properly suited for heterogeneous systems.
+
+With the current design it ranges from cumbersome to impossible to verify whether a memory resource provides allocations
+that are e.g. accessible on device, or whether it can utilize other allocation mechanisms.
+
+To better support asynchronous CUDA `stream-ordered allocations <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#stream-ordered-memory-allocator>`__
+libcu++ provides :cpp:class:`cuda::stream_ref <cuda::stream_ref>` as a wrapper around
+``cudaStream_t``. The definition of ``cuda::stream_ref`` can be found in the ``<cuda/stream>`` header.

cccl_upstream/docs/libcudacxx/extended_api/memory_resource/properties.rst

@@ -0,0 +1,121 @@
+.. _libcudacxx-extended-api-memory-resources-properties:
+
+Properties
+----------
+
+Modern C++ programs use the type system to verify statically known properties of the code. It is undesirable to use
+run-time assertions to verify whether a function that accesses device memory is passed a memory resource that provides
+device-accessible allocations. A property system is provided for this purpose.
+
+To tell the compiler that memory provided by ``my_memory_resource`` is device accessible, use the
+``cuda::mr::device_accessible`` tag type with a free function ``get_property`` to declare that the resource provides
+device-accessible memory.
+
+.. code:: cpp
+
+   struct my_memory_resource {
+       friend constexpr void get_property(const my_memory_resource&, cuda::mr::device_accessible) noexcept {}
+   };
+
+A library can constrain interfaces with ``cuda::has_property`` to require that a passed memory resource provides the
+right kind of memory
+
+`See it on Godbolt <https://godbolt.org/z/5hjoEnerb>`__
+
+.. code:: cpp
+
+   template<class MemoryResource>
+       requires cuda::has_property<MemoryResource, cuda::mr::device_accessible>
+   void function_that_dispatches_to_device(MemoryResource& resource);
+
+If C++20 is not available, the function can instead be constrained via SFINAE
+
+`See it on Godbolt  <https://godbolt.org/z/11sGbr333>`__
+
+.. code:: cpp
+
+   template<class MemoryResource, class = cuda::std::enable_if_t<cuda::has_property<MemoryResource, cuda::mr::device_accessible>>>
+   void function_that_dispatches_to_device(MemoryResource& resource);
+
+For now, libcu++ provides various commonly used properties:
+
+.. list-table::
+   :header-rows: 1
+   :widths: 30 70
+
+   * - Property
+     - Meaning
+   * - ``cuda::mr::device_accessible``
+     - Memory allocated using the resource is accessible from device.
+   * - ``cuda::mr::host_accessible``
+     - Memory allocated using the resource is accessible from host.
+
+``cuda::mr::default_cuda_malloc_alignment`` and ``cuda::mr::default_cuda_malloc_host_alignment`` provide the default
+alignment values used by the CUDA allocation APIs.
+
+More properties may be added as the library and the hardware capabilities evolve. However, a user library is free to
+define custom properties.
+
+Resources may expose a ``default_queries`` member type to advertise a fixed set of properties for APIs that use default
+property queries, such as ``cuda::make_buffer``. Use ``cuda::mr::properties_list<Properties...>`` to define that list.
+
+.. code:: cpp
+
+   struct my_memory_resource {
+     using default_queries = cuda::mr::properties_list<cuda::mr::device_accessible>;
+
+     friend constexpr void get_property(const my_memory_resource&, cuda::mr::device_accessible) noexcept {}
+   };
+
+   static_assert(my_memory_resource::default_queries::has_property(cuda::mr::device_accessible{}));
+
+Note that currently the libcu++ provided properties are stateless. However, properties can also provide stateful
+information that is retrieved via the ``get_property`` free function. In order to communicate the desired type of the
+carried state, a stateful property must define the ``value_type`` alias. A library can constrain interfaces that
+require a stateful property with ``cuda::has_property_with`` as shown in the example below
+
+`See it on Godbolt  <https://godbolt.org/z/11sGbr333>`__
+
+.. code:: cpp
+
+   struct required_alignment{
+       using value_type = std::size_t;
+   };
+   struct my_memory_resource {
+       friend constexpr std::size_t get_property(const my_memory_resource& resource, required_alignment) noexcept { return resource.required_alignment; }
+
+       std::size_t required_alignment;
+   };
+   static_assert(cuda::has_property_with<my_memory_resource, required_alignment, std::size_t>);
+
+   template<class MemoryResource>
+   void* allocate_check_alignment(MemoryResource& resource, std::size_t size) {
+       if constexpr(cuda::has_property_with<MemoryResource, required_alignment, std::size_t>) {
+           return resource.allocate_sync(size, get_property(resource, required_alignment{}));
+       } else {
+           // Use default alignment
+           return resource.allocate_sync(size, 42);
+       }
+   }
+
+In generic code it is often desirable to propagate properties from a base type to a derived type, without knowing the
+base type at all. This common use case is covered by ``cuda::forward_property``, a simple CRTP template.
+
+.. code:: cpp
+
+   template<class MemoryResource>
+   class logging_resource : cuda::forward_property<logging_resource<MemoryResource>, MemoryResource> {
+       MemoryResource base;
+   public:
+       void* allocate_sync(std::size_t size, std::size_t alignment) {
+           std::cout << "allocating\n";
+           return base.allocate_sync(size, alignment);
+       }
+       void deallocate_sync(void* ptr, std::size_t size, std::size_t alignment) noexcept {
+           std::cout << "deallocating\n";
+           return base.deallocate_sync(ptr, size, alignment);
+       }
+
+       MemoryResource& upstream_resource() noexcept { return base; }
+       const MemoryResource& upstream_resource() const noexcept { return base; }
+   };

cccl_upstream/docs/libcudacxx/extended_api/memory_resource/resource.rst

@@ -0,0 +1,154 @@
+.. _libcudacxx-extended-api-memory-resources-resource:
+.. _libcudacxx-extended-api-memory-resources-synchronous-resource:
+
+The ``cuda::mr::resource`` and ``cuda::mr::synchronous_resource`` concepts
+--------------------------------------------------------------------------
+
+The `std::pmr::memory_resource <https://en.cppreference.com/w/cpp/header/memory_resource>`__ feature provides only a
+single ``allocate`` interface, which is sufficient for homogeneous memory systems. However, CUDA provides both
+synchronous and `stream-ordered allocation <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#stream-ordered-memory-allocator>`__.
+
+With `std::pmr::memory_resource <https://en.cppreference.com/w/cpp/header/memory_resource>`__ there is no way to tell
+whether a memory resource can utilize stream-ordered allocations. Even if the application knows it can, there is no way
+to properly tell the memory resource to use stream-ordered allocation. Ideally, this should not be something discovered
+through an assert at run time, but should be checked by the compiler.
+
+Because asynchronous memory management is critical for performance, ``cuda::mr::resource`` includes the stream-ordered interface provided by ``allocate`` / ``deallocate``.
+For cases where stream-ordered allocation is not possible, ``cuda::mr::synchronous_resource`` is provided.
+
+The ``cuda::mr::synchronous_resource`` concept provides basic type checks to ensure that a given memory resource provides the
+expected ``allocate_sync`` / ``deallocate_sync`` interface and is also equality comparable, which covers the whole API surface of
+`std::pmr::memory_resource <https://en.cppreference.com/w/cpp/header/memory_resource>`__.
+See below for different memory resources and potential pitfalls.
+
+To demonstrate, the following example defines several resources, only some of which are valid implementations of the
+``cuda::mr::synchronous_resource`` concept. The ``static_assertions`` will result in compile-time errors for the invalid resources.
+
+.. code:: cpp
+
+   struct valid_resource {
+     void* allocate_sync(std::size_t, std::size_t) { return nullptr; }
+     void deallocate_sync(void*, std::size_t, std::size_t) noexcept {}
+     bool operator==(const valid_resource&) const { return true; }
+     // NOTE: C++20 thankfully added operator rewrite rules so defining operator!= is not required.
+     // However, if compiled with C++14 / C++17, operator != must also be defined.
+     bool operator!=(const valid_resource&) const { return false; }
+   };
+   static_assert(cuda::mr::synchronous_resource<valid_resource>);
+
+   struct invalid_argument {};
+   struct invalid_allocate_argument {
+     void* allocate_sync(invalid_argument, std::size_t) { return nullptr; }
+     void deallocate_sync(void*, std::size_t, std::size_t) noexcept {}
+     bool operator==(const invalid_allocate_argument&) { return true; }
+   };
+   static_assert(!cuda::mr::synchronous_resource<invalid_allocate_argument>);
+
+   struct invalid_allocate_return {
+     int allocate_sync(std::size_t, std::size_t) { return 42; }
+     void deallocate_sync(void*, std::size_t, std::size_t) noexcept {}
+     bool operator==(const invalid_allocate_return&) { return true; }
+   };
+   static_assert(!cuda::mr::synchronous_resource<invalid_allocate_return>);
+
+   struct invalid_deallocate_argument {
+     void* allocate_sync(std::size_t, std::size_t) { return nullptr; }
+     void deallocate_sync(void*, invalid_argument, std::size_t) noexcept {}
+     bool operator==(const invalid_deallocate_argument&) { return true; }
+   };
+   static_assert(!cuda::mr::synchronous_resource<invalid_deallocate_argument>);
+
+   struct non_comparable {
+     void* allocate_sync(std::size_t, std::size_t) { return nullptr; }
+     void deallocate_sync(void*, std::size_t, std::size_t) noexcept {}
+   };
+   static_assert(!cuda::mr::synchronous_resource<non_comparable>);
+
+   struct non_eq_comparable {
+     void* allocate_sync(std::size_t, std::size_t) { return nullptr; }
+     void deallocate_sync(void*, std::size_t, std::size_t) noexcept {}
+     bool operator!=(const non_eq_comparable&) { return false; }
+   };
+   static_assert(!cuda::mr::synchronous_resource<non_eq_comparable>);
+
+In addition to the `std::pmr::memory_resource <https://en.cppreference.com/w/cpp/header/memory_resource>`_ interface the
+``cuda::mr::resource`` concept verifies that a memory resource also satisfies the ``allocate`` /
+``deallocate`` interface. Requiring both the PMR interface and the async interface is a deliberate design decision.
+
+.. code:: cpp
+
+   struct valid_resource {
+     void* allocate_sync(std::size_t, std::size_t) { return nullptr; }
+     void deallocate_sync(void*, std::size_t, std::size_t) noexcept {}
+     void* allocate(cuda::stream_ref, std::size_t, std::size_t) { return nullptr; }
+     void deallocate(cuda::stream_ref, void*, std::size_t, std::size_t) {}
+     bool operator==(const valid_resource&) const { return true; }
+     bool operator!=(const valid_resource&) const { return false; }
+   };
+   static_assert(cuda::mr::resource<valid_resource>);
+
+A library can easily decide whether to use the async interface:
+
+.. code:: cpp
+
+   template<class MemoryResource>
+       requires cuda::mr::synchronous_resource<MemoryResource>
+   void* allocate_maybe_sync(cuda::stream_ref stream, MemoryResource& resource, std::size_t size, std::size_t align) {
+       if constexpr(cuda::mr::resource<MemoryResource>) {
+           return resource.allocate(stream, size, align);
+       } else {
+           return resource.allocate_sync(size, align);
+       }
+   }
+
+.. rubric:: Putting them together
+
+Applications and libraries may want to combine type checks for arbitrary properties with the ``{synchronous_}resource``
+concept. The ``{synchronous_}resource_with`` concept allows checking resources for arbitrary properties.
+
+.. code:: cpp
+
+   struct required_alignment{
+       using value_type = std::size_t;
+   };
+   struct my_memory_resource {
+       void* allocate_sync(std::size_t, std::size_t) { return nullptr; }
+       void deallocate_sync(void*, std::size_t, std::size_t) noexcept {}
+       bool operator==(const my_memory_resource&) const { return true; }
+       bool operator!=(const my_memory_resource&) const { return false; }
+
+       friend constexpr std::size_t get_property(const my_memory_resource& resource, required_alignment) noexcept { return resource.required_alignment; }
+
+       std::size_t required_alignment;
+   };
+
+   constexpr std::size_t my_default_alignment = 16;
+
+   template<class MemoryResource>
+       requires cuda::mr::synchronous_resource<MemoryResource>
+   void* allocate_maybe_sync_check_alignment(MemoryResource& resource, cuda::stream_ref stream, std::size_t size) {
+       if constexpr(cuda::mr::resource_with<MemoryResource, required_alignment>) {
+           return resource.allocate(stream, size, get_property(resource, required_alignment{}));
+       } else if constexpr (cuda::mr::resource<MemoryResource>) {
+           return resource.allocate(stream, size, my_default_alignment);
+       } else if constexpr (cuda::mr::synchronous_resource_with<MemoryResource, required_alignment>) {
+           return resource.allocate_sync(size, get_property(resource, required_alignment{}));
+       } else {
+           return resource.allocate_sync(size, my_default_alignment);
+       }
+   }
+
+   // Potentially more concise
+   template<class MemoryResource>
+       requires cuda::mr::synchronous_resource<MemoryResource>
+   void* allocate_maybe_sync_check_alignment2(MemoryResource& resource, cuda::stream_ref stream, std::size_t size) {
+       std::size_t align = my_default_alignment;
+       if constexpr(cuda::mr::synchronous_resource_with<MemoryResource, required_alignment>) {
+           align = get_property(resource, required_alignment{});
+       }
+       if constexpr(cuda::mr::resource<MemoryResource>) {
+           return resource.allocate(stream, size, align);
+       } else {
+           return resource.allocate_sync(size, align);
+       }
+   }

cccl_upstream/docs/libcudacxx/extended_api/memory_resource/resource_utilities.rst

@@ -0,0 +1,110 @@
+.. _libcudacxx-extended-api-memory-resources-utilities:
+
+Resource utilities
+------------------
+
+The ``cuda::mr`` memory resource system includes utilities that help manage resource lifetime and adapt synchronous
+resources for stream-ordered usage. These utilities complement the type-erased wrappers in :ref:`resource wrappers <libcudacxx-extended-api-memory-resources-resource-ref>`.
+
+shared_resource
+~~~~~~~~~~~~~~~
+.. _libcudacxx-extended-api-memory-resources-shared-resource:
+.. _cccl-runtime-memory-resource-shared-resource:
+
+``cuda::mr::shared_resource`` holds a reference-counted instance of a memory resource, allowing resources to be passed
+around with shared ownership semantics while avoiding lifetime issues. This is useful when multiple objects, like
+:ref:`cuda::buffer <cccl-runtime-buffer-buffer>`, must share the same resource instance.
+
+``cuda::mr::make_shared_resource<Resource>(args...)`` constructs a ``shared_resource<Resource>`` that owns a newly
+constructed ``Resource``. ``Resource`` must satisfy ``cuda::mr::synchronous_resource``. The resulting
+``shared_resource`` supports stream-ordered allocation when ``Resource`` also satisfies ``cuda::mr::resource``.
+
+.. code:: cpp
+
+   #include <cuda/devices>
+   #include <cuda/memory_resource>
+   #include <cuda/stream>
+
+   void use_shared_resource(cuda::stream_ref stream) {
+     // Create a shared resource
+     auto shared_mr = cuda::mr::shared_resource{
+       cuda::std::in_place_type<cuda::device_memory_pool>,
+       cuda::devices[0]
+     };
+
+     // Copy the shared resource (shares ownership)
+     auto shared_mr2 = shared_mr;
+
+     // Both can be used independently
+     void* ptr1 = shared_mr.allocate(stream, 1024, 16);
+     void* ptr2 = shared_mr2.allocate(stream, 2048, 16);
+
+     shared_mr.deallocate(stream, ptr1, 1024, 16);
+     shared_mr2.deallocate(stream, ptr2, 2048, 16);
+     // Resources are automatically cleaned up when last reference is destroyed
+   }
+
+get_memory_resource
+~~~~~~~~~~~~~~~~~~~
+.. _libcudacxx-extended-api-memory-resources-get-memory-resource:
+
+``cuda::mr::get_memory_resource`` is a customization point object that retrieves a memory resource from an object or
+execution environment.
+
+``cuda::mr::memory_resource_base<Derived>`` is a CRTP helper that makes a resource queryable by
+``cuda::mr::get_memory_resource`` when the resource is used as part of a composed execution environment.
+
+It supports three cases:
+
+- If the object is itself a synchronous resource, it returns that object.
+- If the object has a ``get_memory_resource()`` member, it calls that member.
+- Otherwise, if an execution environment supports ``query(cuda::mr::get_memory_resource)``, it calls that query.
+
+.. code:: cpp
+
+   #include <cuda/memory_resource>
+   #include <cuda/stream>
+   #include <cuda/std/type_traits>
+
+   template <class Env>
+   void* allocate_from_env(Env& env, cuda::stream_ref stream, std::size_t size, std::size_t align) {
+     auto&& resource = cuda::mr::get_memory_resource(env);
+     using resource_type = cuda::std::remove_reference_t<decltype(resource)>;
+     if constexpr (cuda::mr::resource<resource_type>) {
+       return resource.allocate(stream, size, align);
+     } else {
+       return resource.allocate_sync(size, align);
+     }
+   }
+
+synchronous_resource_adapter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. _libcudacxx-extended-api-memory-resources-synchronous-adapter:
+.. _cccl-runtime-memory-resource-synchronous-adapter:
+
+``cuda::mr::synchronous_resource_adapter`` adapts a synchronous memory resource to work as a stream-ordered resource.
+If the underlying resource already supports stream-ordered allocation, it passes through the calls. Otherwise, it uses
+synchronous allocation/deallocation with proper stream synchronization.
+
+.. code:: cpp
+
+   #include <cuda/memory_resource>
+   #include <cuda/stream>
+
+   void adapt_sync_resource(cuda::stream_ref stream) {
+     // Create a synchronous resource
+     auto sync_mr = cuda::mr::legacy_pinned_memory_resource{};
+
+     // Adapt it to work with streams
+     auto adapted = cuda::mr::synchronous_resource_adapter{
+       sync_mr
+     };
+
+     // Now can use with stream (will synchronize internally)
+     void* ptr = adapted.allocate(stream, 1024, 16);
+
+     // Use memory...
+
+     // Deallocate (will synchronize stream before deallocation)
+     adapted.deallocate(stream, ptr, 1024, 16);
+   }

cccl_upstream/docs/libcudacxx/extended_api/memory_resource/wrappers.rst

@@ -0,0 +1,253 @@
+.. _libcudacxx-extended-api-memory-resources-wrappers:
+.. _libcudacxx-extended-api-memory-resources-resource-ref:
+.. _cccl-runtime-memory-resource-resource-ref:
+.. _libcudacxx-memory-resource-any-resource:
+.. _libcudacxx-memory-resource-any-async-resource:
+
+Type-erased resource wrappers
+-----------------------------
+
+With the property design depicted in :ref:`cuda::get_property <libcudacxx-extended-api-memory-resources-properties>`,
+a library has flexibility in checking constraints and querying custom properties. However, there is also a cost in
+providing function templates for a potentially wide range of inputs. Depending on the number of different memory
+resources, both compile time and binary size might increase considerably.
+
+The type-erased wrappers let you coalesce such APIs into a single function. Both ``resource_ref`` and ``any_resource``
+preserve property constraints while erasing the concrete resource type.
+
+.. list-table::
+   :widths: 30 20 50
+   :header-rows: 1
+
+   * - Wrapper
+     - Ownership
+     - Typical use
+   * - ``cuda::mr::resource_ref``
+     - Non-owning
+     - Accept references/pointers without extending lifetime
+   * - ``cuda::mr::any_resource``
+     - Owning
+     - Store a resource with the object that uses it
+
+Common usage
+~~~~~~~~~~~~
+
+Both wrappers provide a non-templated API surface for allocation and property queries. Choose which wrapper to use based
+on ownership, then the usage patterns are the same.
+
+``resource_ref`` is constructible from any non-const reference or pointer to a memory resource that satisfies
+``cuda::mr::resource``. ``any_resource`` is constructible from a resource object and takes ownership of it.
+
+Properties may be passed to both wrappers just as with ``cuda::mr::resource_with``.
+
+.. code:: cpp
+
+   void* do_allocate(cuda::mr::resource_ref<> resource, cuda::stream_ref stream, std::size_t size, std::size_t align) {
+       return resource.allocate(stream, size, align);
+   }
+
+   void* do_allocate_owned(cuda::mr::any_resource<> resource, cuda::stream_ref stream, std::size_t size, std::size_t align) {
+       return resource.allocate(stream, size, align);
+   }
+
+   my_memory_resource resource;
+   my_memory_resource* pointer_to_resource = &resource;
+
+   void* from_reference = do_allocate(resource, stream, 1337, 256);
+   void* from_ptr = do_allocate(pointer_to_resource, stream, 1337, 256);
+   void* from_owned = do_allocate_owned(cuda::mr::any_resource<>{resource}, stream, 1337, 256);
+
+resource_ref
+~~~~~~~~~~~~
+.. _libcudacxx-extended-api-memory-resources-resource-ref-wrapper:
+
+``cuda::mr::resource_ref`` is the non-owning, type-erased wrapper. Prefer it when the caller controls the resource
+lifetime.
+
+.. code:: cpp
+
+   struct required_alignment {
+       using value_type = std::size_t;
+   };
+   void* do_allocate_with_alignment(cuda::mr::resource_ref<required_alignment> resource, cuda::stream_ref stream, std::size_t size) {
+       return resource.allocate(stream, size, get_property(resource, required_alignment{}));
+   }
+
+However, the type erasure comes with the cost that arbitrary properties cannot be queried from either wrapper:
+
+.. code:: cpp
+
+   struct required_alignment {
+       using value_type = std::size_t;
+   };
+   void* buggy_allocate_with_alignment(cuda::mr::resource_ref<> resource, cuda::stream_ref stream, std::size_t size) {
+       if constexpr (cuda::has_property<decltype(resource), required_alignment>) { // BUG: This will always be false
+           return resource.allocate(stream, size, get_property(resource, required_alignment{}));
+       } else {
+           return resource.allocate(stream, size, my_default_alignment);
+       }
+   }
+
+So, choose wisely. If your library has a well-defined set of fixed properties that you expect to always be available,
+then ``cuda::mr::{synchronous_}resource_ref`` is an amazing tool to improve compile times and binary size. If you need a
+flexible interface then constraining a template argument through ``cuda::mr::{synchronous_}resource_with`` is the proper solution.
+
+any_resource
+~~~~~~~~~~~~
+.. _libcudacxx-extended-api-memory-resources-any-resource-wrapper:
+.. _libcudacxx-extended-api-memory-resources-any-resource:
+.. _cccl-runtime-memory-resource-any-resource:
+
+``cuda::mr::any_resource`` is the owning counterpart. It is especially suited for containers that must ensure the
+resource outlives the container.
+
+``cuda::mr::make_any_resource<Resource, Properties...>(args...)`` constructs an ``any_resource<Properties...>`` that
+owns a newly constructed ``Resource``. ``Resource`` must satisfy ``cuda::mr::resource_with<Resource, Properties...>``.
+
+.. code:: cpp
+
+   #include <cuda/devices>
+   #include <cuda/memory_resource>
+   #include <cuda/stream>
+
+   void use_any_resource(cuda::stream_ref stream) {
+     // Wrap a device memory resource
+     cuda::mr::any_resource<cuda::mr::device_accessible> resource{
+       cuda::device_default_memory_pool(cuda::devices[0])
+     };
+
+     // Allocate memory
+     void* ptr = resource.allocate(stream, 1024, 16);
+
+     // Use memory...
+
+     // Deallocate
+     resource.deallocate(stream, ptr, 1024, 16);
+   }
+
+Synchronous variants
+~~~~~~~~~~~~~~~~~~~~
+
+The synchronous wrappers mirror the same ownership split: ``synchronous_resource_ref`` is non-owning, while
+``any_synchronous_resource`` owns the resource instance.
+
+``cuda::mr::any_synchronous_resource``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. _libcudacxx-extended-api-memory-resources-any-synchronous-resource:
+.. _cccl-runtime-memory-resource-any-synchronous-resource:
+
+``cuda::mr::any_synchronous_resource`` is a type-erased wrapper for resources that satisfy
+``cuda::mr::synchronous_resource``. It can be constructed from an ``any_resource``, but only exposes synchronous
+allocation/deallocation.
+
+``cuda::mr::make_any_synchronous_resource<Resource, Properties...>(args...)`` constructs an
+``any_synchronous_resource<Properties...>`` that owns a newly constructed ``Resource``. ``Resource`` must satisfy
+``cuda::mr::synchronous_resource_with<Resource, Properties...>``.
+
+.. code:: cpp
+
+   #include <cuda/memory_resource>
+
+   void use_sync_resource() {
+     cuda::mr::any_synchronous_resource<cuda::mr::host_accessible> resource{
+       cuda::mr::legacy_pinned_memory_resource{}
+     };
+
+     // Synchronous allocation
+     void* ptr = resource.allocate_sync(1024, 16);
+
+     // Use memory...
+
+     // Synchronous deallocation
+     resource.deallocate_sync(ptr, 1024, 16);
+   }
+
+resource_cast
+~~~~~~~~~~~~~
+.. _libcudacxx-extended-api-memory-resources-resource-cast:
+
+``cuda::mr::resource_cast<T>(&resource)`` returns a pointer to the concrete resource stored in a type-erased wrapper
+when the stored resource has type ``T``. It returns ``nullptr`` when the stored type does not match.
+
+This is similar to pointer-form ``dynamic_cast<T*>`` in that callers must check for ``nullptr`` before using the
+result. Unlike ``dynamic_cast``, ``resource_cast`` matches the exact type stored in the wrapper.
+
+``resource_cast`` works with ``any_resource``, ``any_synchronous_resource``, ``resource_ref``, and
+``synchronous_resource_ref``.
+
+.. code:: cpp
+
+   #include <cuda/devices>
+   #include <cuda/memory_resource>
+   #include <cuda/stream>
+
+   void use_resource_cast(cuda::stream_ref stream) {
+     cuda::mr::any_resource<cuda::mr::device_accessible> resource{
+       cuda::device_default_memory_pool(cuda::devices[0])
+     };
+
+     if (auto* pool = cuda::mr::resource_cast<cuda::device_memory_pool_ref>(&resource)) {
+       pool->trim_to(1024 * 1024);
+
+       void* ptr = pool->allocate(stream, 1024, 16);
+       pool->deallocate(stream, ptr, 1024, 16);
+     }
+   }
+
+try_get_property
+~~~~~~~~~~~~~~~~
+.. _libcudacxx-extended-api-memory-resources-try-get-property:
+
+Type-erased wrapper conversions can discard properties from the wrapper's type. ``try_get_property(resource, property)``
+can query a property that was present before such a conversion, even when it is not listed in the resulting wrapper's
+property set. For stateless properties, it returns ``bool``. For stateful properties, it returns
+``cuda::std::optional<Property::value_type>``.
+
+.. code:: cpp
+
+   #include <cuda/memory_resource>
+
+   bool is_device_accessible(cuda::mr::any_synchronous_resource<> resource) {
+     return try_get_property(resource, cuda::mr::device_accessible{});
+   }
+
+dynamic_resource_cast
+~~~~~~~~~~~~~~~~~~~~~
+.. _libcudacxx-extended-api-memory-resources-dynamic-resource-cast:
+
+``cuda::mr::dynamic_resource_cast<Properties...>(resource)`` casts a type-erased resource wrapper to the same wrapper
+kind with a different property set. It checks the stored resource at run time and throws if the stored resource does
+not support the requested properties.
+
+.. code:: cpp
+
+   #include <cuda/memory_resource>
+
+   cuda::mr::any_resource<cuda::mr::device_accessible>
+   require_device_accessible(cuda::mr::any_resource<> resource) {
+     return cuda::mr::dynamic_resource_cast<cuda::mr::device_accessible>(cuda::std::move(resource));
+   }
+
+``cuda::mr::synchronous_resource_ref``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+.. _cccl-runtime-memory-resource-synchronous-resource-ref:
+
+``cuda::mr::synchronous_resource_ref`` provides the same type-erased reference behavior as ``resource_ref``, but only
+exposes synchronous allocation and deallocation.
+
+.. code:: cpp
+
+   #include <cuda/memory_resource>
+
+   // indicates that the resource must be host-accessible
+   void use_sync_ref(
+     cuda::mr::synchronous_resource_ref<cuda::mr::host_accessible> resource
+   ) {
+     // Synchronous allocation
+     void* ptr = resource.allocate_sync(1024, 16);
+
+     // Use memory...
+
+     // Synchronous deallocation
+     resource.deallocate_sync(ptr, 1024, 16);
+   }

cccl_upstream/docs/libcudacxx/extended_api/numeric.rst

@@ -0,0 +1,96 @@
+.. _libcudacxx-extended-api-numeric:
+
+Numeric
+========
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   numeric/add_overflow
+   numeric/saturating_add_overflow
+   numeric/div_overflow
+   numeric/saturating_div_overflow
+   numeric/isclose
+   numeric/mul_overflow
+   numeric/saturating_mul_overflow
+   numeric/narrow
+   numeric/overflow_cast
+   numeric/overflow_result
+   numeric/saturating_overflow_cast
+   numeric/sub_overflow
+   numeric/saturating_sub_overflow
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`cuda::narrow <libcudacxx-extended-api-numeric-narrow>`
+     - Casts a value and checks whether the value has changed
+     - CCCL 3.1.0
+     - CUDA 13.1
+
+   * - :ref:`cuda::overflow_result <libcudacxx-extended-api-numeric-overflow_result>`
+     - Represents the result of arithmetic operations that may overflow
+     - CCCL 3.0.0
+     - CUDA 13.0
+
+   * - :ref:`cuda::overflow_cast <libcudacxx-extended-api-numeric-overflow_cast>`
+     - Casts a value with overflow checking
+     - CCCL 3.0.0
+     - CUDA 13.0
+
+   * - :ref:`cuda::add_overflow <libcudacxx-extended-api-numeric-add_overflow>`
+     - Performs addition with overflow checking
+     - CCCL 3.2.0
+     - CUDA 13.2
+
+   * - :ref:`cuda::sub_overflow <libcudacxx-extended-api-numeric-sub_overflow>`
+     - Performs subtraction with overflow checking
+     - CCCL 3.2.0
+     - CUDA 13.2
+
+   * - :ref:`cuda::div_overflow <libcudacxx-extended-api-numeric-div_overflow>`
+     - Performs division with overflow checking
+     - CCCL 3.2.0
+     - CUDA 13.2
+
+   * - :ref:`cuda::isclose <libcudacxx-extended-api-numeric-isclose>`
+     - Checks whether two values are approximately equal
+     - CCCL 3.5.0
+     - CUDA 13.5
+
+   * - :ref:`cuda::mul_overflow <libcudacxx-extended-api-numeric-mul_overflow>`
+     - Performs multiplication with overflow checking
+     - CCCL 3.4.0
+     - CUDA 13.4
+
+   * - :ref:`cuda::saturating_overflow_cast <libcudacxx-extended-api-numeric-saturating_overflow_cast>`
+     - Performs saturating cast of a value with overflow checking
+     - CCCL 3.4.0
+     - CUDA 13.4
+
+   * - :ref:`cuda::saturating_add_overflow <libcudacxx-extended-api-numeric-saturating_add_overflow>`
+     - Performs saturating addition with overflow checking
+     - CCCL 3.4.0
+     - CUDA 13.4
+
+   * - :ref:`cuda::saturating_sub_overflow <libcudacxx-extended-api-numeric-saturating_sub_overflow>`
+     - Performs saturating subtraction with overflow checking
+     - CCCL 3.4.0
+     - CUDA 13.4
+
+   * - :ref:`cuda::saturating_div_overflow <libcudacxx-extended-api-numeric-saturating_div_overflow>`
+     - Performs saturating division with overflow checking
+     - CCCL 3.4.0
+     - CUDA 13.4
+
+   * - :ref:`cuda::saturating_mul_overflow <libcudacxx-extended-api-numeric-saturating_mul_overflow>`
+     - Performs saturating multiplication with overflow checking
+     - CCCL 3.4.0
+     - CUDA 13.4

cccl_upstream/docs/libcudacxx/extended_api/numeric/add_overflow.rst

@@ -0,0 +1,89 @@
+.. _libcudacxx-extended-api-numeric-add_overflow:
+
+``cuda::add_overflow``
+======================
+
+Defined in ``<cuda/numeric>`` header.
+
+.. code:: cpp
+
+   namespace cuda {
+
+   template <class T>
+   struct overflow_result;
+
+   template <class Result = /*unspecified*/, class Lhs, class Rhs>
+   [[nodiscard]] __host__ __device__ constexpr
+   overflow_result</*see-below*/> add_overflow(Lhs lhs, Rhs rhs) noexcept; // (1)
+
+   template <class Result, class Lhs, class Rhs>
+   [[nodiscard]] __host__ __device__ constexpr
+   bool add_overflow(Result& result, Lhs lhs, Rhs rhs) noexcept; // (2)
+
+   } // namespace cuda
+
+The function ``cuda::add_overflow`` performs addition of two values ``lhs`` and ``rhs`` with overflow checking. The result is the same as if the operands were first promoted to an infinite precision signed type, added together and the result truncated to the type of the return value.
+
+**Parameters**
+
+- ``result``: The result of the addition (2).
+- ``lhs``: The left-hand side operand (1, 2).
+- ``rhs``: The right-hand side operand (1, 2).
+
+**Return value**
+
+1. Returns an :ref:`overflow_result <libcudacxx-extended-api-numeric-overflow_result>` object  containing the result of the addition and a boolean indicating whether an overflow or underflow occurred. If the ``Result`` type is specified, it will be used as the type of the result, otherwise the common type of ``Lhs`` and ``Rhs`` is used.
+2. Returns ``true`` if an overflow or underflow occurred, ``false`` otherwise.
+
+**Constraints**
+
+- ``Result``, ``Lhs``, and ``Rhs`` must be `integer types <https://eel.is/c++draft/basic.fundamental#1>`_.
+
+**Performance considerations**
+
+- No overflow checking is required if ``Lhs +  Rhs`` is always representable with the ``Result`` type.
+- Computation is generally faster when ``Lhs``, ``Rhs``, and ``Result`` have the same signedness.
+- Unsigned computations are generally faster than signed computations.
+- The function uses PTX ``asm`` on device and compiler intrinsics on host whenever possible.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/numeric>
+    #include <cuda/std/cassert>
+    #include <cuda/std/limits>
+
+    __global__ void kernel()
+    {
+        constexpr auto int_max = cuda::std::numeric_limits<int>::max();
+        constexpr auto int_min = cuda::std::numeric_limits<int>::min();
+
+        // cuda::add_overflow(lhs, rhs) returning common type of lhs and rhs
+        // 'result' is evaluated to true if an overflow occurred, false otherwise
+        if (auto result = cuda::add_overflow(1, int_max))
+        {
+            assert(result.value == int_min);
+        }
+
+        // cuda::add_overflow<Result>(lhs, rhs) with explicit return type
+        auto result = cuda::add_overflow<long long>(-1, int_min)
+        assert(!result.overflow); // no overflow
+        assert(result.value == static_cast<long long>(int_min) + (-1ll));
+
+        unsigned result{};
+        // cuda::add_overflow(result, lhs, rhs) with bool return type
+        if (!cuda::add_overflow(result, 1, int_max))
+        {
+            assert(result.value == static_cast<unsigned>(int_max) + 1u);
+        }
+    }
+
+    int main()
+    {
+        kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/PPT17ozx6>`_

cccl_upstream/docs/libcudacxx/extended_api/numeric/div_overflow.rst

@@ -0,0 +1,92 @@
+.. _libcudacxx-extended-api-numeric-div_overflow:
+
+``cuda::div_overflow``
+======================
+
+Defined in ``<cuda/numeric>`` header.
+
+.. code:: cpp
+
+   namespace cuda {
+
+   template <class T>
+   struct overflow_result;
+
+   template <class Result = /*unspecified*/, class Lhs, class Rhs>
+   [[nodiscard]] __host__ __device__ constexpr
+   overflow_result</*see-below*/> div_overflow(Lhs lhs, Rhs rhs) noexcept; // (1)
+
+   template <class Result, class Lhs, class Rhs>
+   [[nodiscard]] __host__ __device__ constexpr
+   bool div_overflow(Result& result, Lhs lhs, Rhs rhs) noexcept; // (2)
+
+   } // namespace cuda
+
+The function ``cuda::div_overflow`` performs integer division of ``lhs`` by ``rhs`` with overflow and error detection. The result is the same as if the operands were first promoted to an infinite precision signed type, divided, and the result truncated to the type of the return value.
+
+**Parameters**
+
+- ``result``: Receives the quotient when no overflow is detected (2).
+- ``lhs``: The dividend (1, 2).
+- ``rhs``: The divisor (1, 2).
+
+**Return value**
+
+1. Returns an :ref:`overflow_result <libcudacxx-extended-api-numeric-overflow_result>` object containing the computed quotient and a boolean flag indicating whether an overflow or underflow occurred. If the ``Result`` type is specified, it will be used as the type of the result, otherwise the common type of ``Lhs`` and ``Rhs`` is used.
+2. Returns ``true`` if an overflow or underflow occurred. When ``false`` is returned, the computed quotient is stored in ``result``.
+
+**Preconditions**
+
+- ``rhs != 0``
+
+**Constraints**
+
+- ``Result``, ``Lhs``, and ``Rhs`` must be `integer types <https://eel.is/c++draft/basic.fundamental#1>`_.
+
+**Notes**
+
+- For signed types, ``numeric_limits<Lhs>::min()`` divided by ``-1`` triggers overflow when ``Result`` matches the operand types.
+
+**Performance considerations**
+
+- No overflow checking if ``Lhs / Rhs`` is always  representable with the ``Result`` type.
+- The computation with unsigned types is faster than signed types.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/numeric>
+    #include <cuda/std/cassert>
+    #include <cuda/std/limits>
+
+    __global__ void kernel()
+    {
+        constexpr auto int_min = cuda::std::numeric_limits<int>::min();
+
+        // cuda::div_overflow(lhs, rhs) returning the common type of the operands
+        if (auto result = cuda::div_overflow(-1, int_min))
+        {
+            assert(result.overflow);
+        }
+
+        // cuda::div_overflow<Result>(lhs, rhs) with an explicitly wider result type
+        auto wide = cuda::div_overflow<long long>(-1, int_min);
+        assert(!wide.overflow);
+        assert(wide.value == 0);
+
+        unsigned quotient{};
+        // cuda::div_overflow(result, lhs, rhs) with bool return type
+        bool overflow = cuda::div_overflow(quotient, 10u, 2u);
+        assert(!overflow);
+        assert(quotient == 5u);
+    }
+
+    int main()
+    {
+        kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/dYG3dWss5>`_

cccl_upstream/docs/libcudacxx/extended_api/numeric/isclose.rst

@@ -0,0 +1,103 @@
+.. _libcudacxx-extended-api-numeric-isclose:
+
+``cuda::isclose``
+=================
+
+Defined in ``<cuda/numeric>`` header.
+
+.. code:: cpp
+
+   namespace cuda {
+
+   template <class T>
+   [[nodiscard]] __host__ __device__
+   bool isclose(T lhs, T rhs) noexcept;
+
+   template <class T>
+   [[nodiscard]] __host__ __device__
+   bool isclose(T lhs, T rhs, float relative_tol) noexcept;
+
+   template <class T>
+   [[nodiscard]] __host__ __device__
+   bool isclose(T lhs, T rhs, float relative_tol, T absolute_tol) noexcept;
+
+   template <class Complex>
+   [[nodiscard]] __host__ __device__
+   bool isclose(const Complex& lhs, const Complex& rhs) noexcept;
+
+   template <class Complex>
+   [[nodiscard]] __host__ __device__
+   bool isclose(const Complex& lhs, const Complex& rhs, float relative_tol) noexcept;
+
+   template <class Complex, class AbsTol>
+   [[nodiscard]] __host__ __device__
+   bool isclose(const Complex& lhs,
+                const Complex& rhs,
+                float  relative_tol,
+                AbsTol absolute_tol) noexcept;
+
+   } // namespace cuda
+
+``cuda::isclose`` checks whether two values are approximately equal using the weak symmetric comparison in a similar manner to `PEP 485 <https://peps.python.org/pep-0485/>`_:
+
+.. code:: cpp
+
+   abs(lhs - rhs) <= max(absolute_tol, relative_tol * max(abs(lhs), abs(rhs)))
+
+- For integral operands, ``relative_tol`` is interpreted as its exact binary floating-point value. Comparing it to the integral difference is equivalent to rounding the relative threshold down to the nearest integer.
+- The overloads without ``relative_tol`` use a default relative tolerance based on half of available digits of accuracy. The default relative tolerance for integer types is 0.
+- The overloads without ``absolute_tol`` use ``absolute_tol == 0``.
+
+**Parameters**
+
+- ``lhs``: The first value to compare.
+- ``rhs``: The second value to compare.
+- ``relative_tol``: The relative tolerance. Passing ``0`` performs a purely absolute tolerance check when ``absolute_tol`` is non-zero.
+- ``absolute_tol``: The absolute tolerance. This is useful for comparisons near zero.
+
+**Return value**
+
+- Returns ``true`` if ``lhs`` and ``rhs`` are close to each other, otherwise returns ``false``.
+
+**Preconditions**
+
+- ``relative_tol``: Must be in the range ``[0.0, 1.0]``.
+- ``absolute_tol``: Must be finite and non-negative.
+
+**Constraints**
+
+- Scalar overloads require ``lhs``, ``rhs``, ``absolute_tol`` to have the same arithmetic type (integer or floating point).
+- Complex overloads accept ``cuda::std::complex<T>`` and ``std::complex<T>`` operands.
+- ``AbsTol`` must be the same type as the complex value type.
+
+**Special values**
+
+- ``NaN`` is never close to any value, including another ``NaN``.
+- Infinity and negative infinity are only close to themselves.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/numeric>
+    #include <cuda/std/cassert>
+    #include <cuda/std/complex>
+
+    __global__ void kernel()
+    {
+        assert(cuda::isclose( 1.0f, 1.0f + 5e-6f));
+        assert(!cuda::isclose(1.0f, 1.0f + 2e-5f));
+
+        assert(cuda::isclose( 0.0f, 1e-12f, 0.0f, 1e-12f));
+
+        cuda::std::complex<float> z1{1.0f, 1.0f};
+        cuda::std::complex<float> z2{2.0f, 0.0f};
+        assert(cuda::isclose(z1, z2, 0.75f));
+    }
+
+    int main()
+    {
+        kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+    }

cccl_upstream/docs/libcudacxx/extended_api/numeric/mul_overflow.rst

@@ -0,0 +1,88 @@
+.. _libcudacxx-extended-api-numeric-mul_overflow:
+
+``cuda::mul_overflow``
+======================
+
+Defined in ``<cuda/numeric>`` header.
+
+.. code:: cpp
+
+   namespace cuda {
+
+   template <class T>
+   struct overflow_result;
+
+   template <class Result = /*unspecified*/, class Lhs, class Rhs>
+   [[nodiscard]] __host__ __device__ constexpr
+   overflow_result</*see-below*/> mul_overflow(Lhs lhs, Rhs rhs) noexcept; // (1)
+
+   template <class Result, class Lhs, class Rhs>
+   [[nodiscard]] __host__ __device__ constexpr
+   bool mul_overflow(Result& result, Lhs lhs, Rhs rhs) noexcept; // (2)
+
+   } // namespace cuda
+
+The function ``cuda::mul_overflow`` performs multiplication of two values ``lhs`` and ``rhs`` with overflow checking. The result is the same as if the operands were first promoted to an infinite precision signed type, multiplied together and the result truncated to the type of the return value.
+
+**Parameters**
+
+- ``result``: The result of the multiplication (2).
+- ``lhs``: The left-hand side operand (1, 2).
+- ``rhs``: The right-hand side operand (1, 2).
+
+**Return value**
+
+1. Returns an :ref:`overflow_result <libcudacxx-extended-api-numeric-overflow_result>` object  containing the result of the multiplication and a boolean indicating whether an overflow or underflow occurred. If the ``Result`` type is specified, it will be used as the type of the result, otherwise the common type of ``Lhs`` and ``Rhs`` is used.
+2. Returns ``true`` if an overflow or underflow occurred, ``false`` otherwise.
+
+**Constraints**
+
+- ``Result``, ``Lhs``, and ``Rhs`` must be `integer types <https://eel.is/c++draft/basic.fundamental#1>`_.
+
+**Performance considerations**
+
+- No overflow checking is required if ``Lhs +  Rhs`` is always representable with the ``Result`` type.
+- Computation is generally faster when ``Lhs``, ``Rhs``, and ``Result`` have the same signedness.
+- Unsigned computations are generally faster than signed computations.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/numeric>
+    #include <cuda/std/cassert>
+    #include <cuda/std/limits>
+
+    __global__ void kernel()
+    {
+        constexpr auto int_max = cuda::std::numeric_limits<int>::max();
+        constexpr auto int_min = cuda::std::numeric_limits<int>::min();
+
+        // cuda::mul_overflow(lhs, rhs) returning common type of lhs and rhs
+        // 'result' is evaluated to true if an overflow occurred, false otherwise
+        if (auto result1 = cuda::mul_overflow(-1, int_min))
+        {
+            assert(result1.value == int_min);
+        }
+
+        // cuda::mul_overflow<Result>(lhs, rhs) with explicit return type
+        auto [result2, overflow2] = cuda::mul_overflow<long long>(-1, int_min);
+        assert(!overflow2); // no overflow
+        assert(result2 == static_cast<long long>(int_min) * (-1ll));
+
+        unsigned result3{};
+        // cuda::mul_overflow(result, lhs, rhs) with bool return type
+        if (!cuda::mul_overflow(result3, 2, int_max))
+        {
+            assert(result3 == static_cast<unsigned>(int_max) * 2u);
+        }
+    }
+
+    int main()
+    {
+        kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/hTME8v6rK>`_

cccl_upstream/docs/libcudacxx/extended_api/numeric/narrow.rst

@@ -0,0 +1,55 @@
+.. _libcudacxx-extended-api-numeric-narrow:
+
+``cuda::narrow``
+=====================
+
+.. code:: cpp
+
+   struct narrowing_error;
+
+   template <typename To, typename From>
+   [[nodiscard]] constexpr
+   To narrow(From from);
+
+   template <typename To, typename From>
+   [[nodiscard]] constexpr
+   To narrow_cast(From&& __from) noexcept;
+
+
+Both functions use a ``static_cast`` to cast the value ``from`` to type ``To``.
+``From`` needs to be convertible to ``To``, and implement ``operator!=``.
+``cuda::narrow`` additionally checks whether the value has changed,
+and if so, throws ``cuda::narrowing_error`` in host code and traps in device code.
+In this case, ``To`` additionally needs to be convertible to ``From``.
+``cuda::narrow_cast`` does not perform such a check (it's a plain cast) and is just intended to show
+that narrowing and a potential change of the value is intended.
+The functions are modelled after ``gsl::narrow`` and  ``gsl::narrow_cast``.
+See also the C++ Core Guidelines
+`ES.46 <https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Res-narrowing>`_ and
+`ES.49 <https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Res-casts-named>`_.
+
+
+Example
+-------
+
+.. code:: cpp
+
+    #include <cuda/numeric>
+
+    __global__ void kernel(size_t n) {
+        unsigned int r1 = cuda::narrow<unsigned int>(n); // traps
+
+        unsigned int r2 = cuda::narrow_cast<unsigned int>(n); // truncation of value is intended
+    }
+
+    void host() {
+        unsigned char r1 = cuda::narrow<unsigned char>( 200); // ok
+        unsigned char r2 = cuda::narrow<unsigned char>( 300); // throws narrowing_error
+        unsigned int  r3 = cuda::narrow<unsigned int >(-100); // throws narrowing_error
+
+        unsigned char r4 = cuda::narrow_cast<unsigned char>(300); // truncation of value is intended
+
+        kernel<<<1, 1>>>(2LL << 35); // size larger than unsigned int
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/ahcqv6joY>`_

cccl_upstream/docs/libcudacxx/extended_api/numeric/overflow_cast.rst

@@ -0,0 +1,68 @@
+.. _libcudacxx-extended-api-numeric-overflow_cast:
+
+``cuda::overflow_cast``
+==========================
+
+.. code:: cpp
+
+   template <class T>
+   struct overflow_result;
+
+   template <class To, class From>
+   [[nodiscard]] __host__ __device__ inline constexpr
+   overflow_result<To> overflow_cast(From from) noexcept;
+
+The function ``cuda::overflow_cast`` casts a value of type ``From`` to type ``To`` with overflow checking.
+
+**Parameters**
+
+- ``from``: The value to be casted.
+
+**Return value**
+
+- Returns an :ref:`overflow_result <libcudacxx-extended-api-numeric-overflow_result>` object that contains the result of the cast and a boolean indicating whether an overflow occurred.
+
+**Constraints**
+
+- ``To`` and ``From`` must be `integer types <https://eel.is/c++draft/basic.fundamental#1>`_.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/numeric>
+    #include <cuda/std/cassert>
+    #include <cuda/std/limits>
+
+    __global__ void kernel()
+    {
+        constexpr auto int_max = cuda::std::numeric_limits<int>::max();
+        constexpr auto int_min = cuda::std::numeric_limits<int>::min();
+
+        if (auto result = cuda::overflow_cast<unsigned>(int_max))
+        {
+            assert(false); // Should not be reached
+        }
+        else
+        {
+            assert(result.value == static_cast<unsigned>(int_max));
+        }
+
+        if (auto result = cuda::overflow_cast<unsigned>(int_min))
+        {
+            assert(result.value == static_cast<unsigned>(int_min));
+        }
+        else
+        {
+            assert(false); // Should not be reached
+        }
+    }
+
+    int main()
+    {
+        kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/vK3WMMs1n>`_

cccl_upstream/docs/libcudacxx/extended_api/numeric/overflow_result.rst

@@ -0,0 +1,38 @@
+.. _libcudacxx-extended-api-numeric-overflow_result:
+
+``cuda::overflow_result``
+=========================
+
+.. code:: cpp
+
+   template <class T>
+   struct overflow_result
+   {
+        T    value;
+        bool overflow;
+
+        __host__ __device__
+        constexpr explicit operator bool() const noexcept;
+   };
+
+The ``overflow_result`` struct is used to represent the result of arithmetic operations that may overflow. It contains the following members:
+
+- ``value``: The result of the operation of type ``T``.
+- ``overflow``: A boolean indicating whether an overflow occurred during the operation.
+
+The ``operator bool()`` returns ``true`` if an overflow occurred, and ``false`` otherwise.
+It can be used in conditional expressions to check whether an overflow occurred.
+
+Example:
+
+.. code:: cpp
+
+    auto result = /* overflow operation */;
+    if (result)
+    {
+        // Overflow occurred
+    }
+
+**Constraints**
+
+- ``T`` must be an integer type.

cccl_upstream/docs/libcudacxx/extended_api/numeric/saturating_add_overflow.rst

@@ -0,0 +1,78 @@
+.. _libcudacxx-extended-api-numeric-saturating_add_overflow:
+
+``cuda::saturating_add_overflow``
+=================================
+
+Defined in ``<cuda/numeric>`` header.
+
+.. code:: cpp
+
+   namespace cuda {
+
+   template <class T>
+   struct overflow_result;
+
+   template <class T>
+   [[nodiscard]] __host__ __device__ constexpr
+   overflow_result<T> saturating_add_overflow(T lhs, T rhs) noexcept; // (1)
+
+   template <class T>
+   [[nodiscard]] __host__ __device__ constexpr
+   bool saturating_add_overflow(T& result, T lhs, T rhs) noexcept; // (2)
+
+   } // namespace cuda
+
+The function ``cuda::saturating_add_overflow`` performs saturating addition of two values ``lhs`` and ``rhs`` with overflow checking.
+
+**Parameters**
+
+- ``result``: The result of the saturating addition. (2)
+- ``lhs``: The left-hand side operand. (1, 2)
+- ``rhs``: The right-hand side operand. (1, 2)
+
+**Return value**
+
+1. Returns an :ref:`overflow_result <libcudacxx-extended-api-numeric-overflow_result>` object containing the result of the saturating addition and a boolean indicating whether an overflow or underflow occurred.
+2. Returns ``true`` if an overflow or underflow occurred, ``false`` otherwise.
+
+**Constraints**
+
+- ``T`` must be an `integer type <https://eel.is/c++draft/basic.fundamental#1>`_.
+
+**Performance considerations**
+
+- Functionality is implemented by correcting the ``cuda::add_overflow`` result in case of overflow/underflow.
+- Unsigned computations are generally faster than signed computations.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/numeric>
+    #include <cuda/std/cassert>
+    #include <cuda/std/limits>
+
+    __global__ void kernel()
+    {
+        constexpr auto int_max = cuda::std::numeric_limits<int>::max();
+
+        const auto result = cuda::saturating_add_overflow(1, int_max); // saturated
+        assert(result.value == int_max);
+        assert(result.overflow);
+
+        int value;
+        if (cuda::saturating_add_overflow(value, 42, 1024))
+        {
+            assert(false); // shouldn't be reached
+        }
+        assert(value == 1066);
+    }
+
+    int main()
+    {
+        kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/f4jrMb5cW>`_

cccl_upstream/docs/libcudacxx/extended_api/numeric/saturating_div_overflow.rst

@@ -0,0 +1,83 @@
+.. _libcudacxx-extended-api-numeric-saturating_div_overflow:
+
+``cuda::saturating_div_overflow``
+=================================
+
+Defined in ``<cuda/numeric>`` header.
+
+.. code:: cpp
+
+   namespace cuda {
+
+   template <class T>
+   struct overflow_result;
+
+   template <class T>
+   [[nodiscard]] __host__ __device__ constexpr
+   overflow_result<T> saturating_div_overflow(T lhs, T rhs) noexcept; // (1)
+
+   template <class T>
+   [[nodiscard]] __host__ __device__ constexpr
+   bool saturating_div_overflow(T& result, T lhs, T rhs) noexcept; // (2)
+
+   } // namespace cuda
+
+The function ``cuda::saturating_div_overflow`` performs saturating integer division of ``lhs`` by ``rhs`` with overflow and error detection.
+
+**Parameters**
+
+- ``result``: Result of the saturating integer division. (2)
+- ``lhs``: The dividend. (1, 2)
+- ``rhs``: The divisor. (1, 2)
+
+**Return value**
+
+1. Returns an :ref:`overflow_result <libcudacxx-extended-api-numeric-overflow_result>` object containing the result of saturating integer division and a boolean flag indicating whether an overflow or underflow occurred.
+2. Returns ``true`` if an overflow or underflow occurred.
+
+**Preconditions**
+
+- ``rhs != 0``
+
+**Constraints**
+
+- ``T`` must be an `integer type <https://eel.is/c++draft/basic.fundamental#1>`_.
+
+**Performance considerations**
+
+- Functionality is implemented by correcting the ``cuda::div_overflow`` result in case of overflow/underflow.
+- Unsigned computations are generally faster than signed computations.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/numeric>
+    #include <cuda/std/cassert>
+    #include <cuda/std/limits>
+
+    __global__ void kernel()
+    {
+        constexpr auto int_max = cuda::std::numeric_limits<int>::max();
+        constexpr auto int_min = cuda::std::numeric_limits<int>::min();
+
+        const auto result = cuda::saturating_div_overflow(int_min, -1); // saturated
+        assert(result.value == int_max);
+        assert(result.overflow);
+
+        int value;
+        if (cuda::saturating_div_overflow(value, 256, 8))
+        {
+            assert(false); // shouldn't be reached
+        }
+        assert(value == 32);
+    }
+
+    int main()
+    {
+        kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/T91jMKEjY>`_

cccl_upstream/docs/libcudacxx/extended_api/numeric/saturating_mul_overflow.rst

@@ -0,0 +1,79 @@
+.. _libcudacxx-extended-api-numeric-saturating_mul_overflow:
+
+``cuda::saturating_mul_overflow``
+=================================
+
+Defined in ``<cuda/numeric>`` header.
+
+.. code:: cpp
+
+   namespace cuda {
+
+   template <class T>
+   struct overflow_result;
+
+   template <class T>
+   [[nodiscard]] __host__ __device__ constexpr
+   overflow_result<T> saturating_mul_overflow(T lhs, T rhs) noexcept; // (1)
+
+   template <class T>
+   [[nodiscard]] __host__ __device__ constexpr
+   bool saturating_mul_overflow(T& result, T lhs, T rhs) noexcept; // (2)
+
+   } // namespace cuda
+
+The function ``cuda::saturating_mul_overflow`` performs saturating multiplication of two values ``lhs`` and ``rhs`` with overflow checking.
+
+**Parameters**
+
+- ``result``: The result of the saturating multiplication (2).
+- ``lhs``: The left-hand side operand (1, 2).
+- ``rhs``: The right-hand side operand (1, 2).
+
+**Return value**
+
+1. Returns an :ref:`overflow_result <libcudacxx-extended-api-numeric-overflow_result>` object  containing the result of the saturating multiplication and a boolean indicating whether an overflow or underflow occurred.
+2. Returns ``true`` if an overflow or underflow occurred, ``false`` otherwise.
+
+**Constraints**
+
+- ``T`` must be an `integer type <https://eel.is/c++draft/basic.fundamental#1>`_.
+
+**Performance considerations**
+
+- Functionality is implemented by correcting the ``cuda::mul_overflow`` result in case of overflow/underflow.
+- Unsigned computations are generally faster than signed computations.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/numeric>
+    #include <cuda/std/cassert>
+    #include <cuda/std/limits>
+
+    __global__ void kernel()
+    {
+        constexpr auto int_max = cuda::std::numeric_limits<int>::max();
+        constexpr auto int_min = cuda::std::numeric_limits<int>::min();
+
+        const auto result = cuda::saturating_mul_overflow(int_max, int_min); // saturated
+        assert(result.value == int_min);
+        assert(result.overflow);
+
+        int value;
+        if (cuda::saturating_mul_overflow(value, 4, 8))
+        {
+            assert(false); // shouldn't be reached
+        }
+        assert(value == 32);
+    }
+
+    int main()
+    {
+        kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/chrG67YWT>`_

cccl_upstream/docs/libcudacxx/extended_api/numeric/saturating_overflow_cast.rst

@@ -0,0 +1,68 @@
+.. _libcudacxx-extended-api-numeric-saturating_overflow_cast:
+
+``cuda::overflow_cast``
+==========================
+
+.. code:: cpp
+
+   template <class T>
+   struct overflow_result;
+
+   template <class To, class From>
+   [[nodiscard]] __host__ __device__ inline constexpr
+   overflow_result<To> saturating_overflow_cast(From from) noexcept;
+
+The function ``cuda::saturating_overflow_cast`` does saturating cast of a value of type ``From`` to type ``To`` with overflow checking.
+
+**Parameters**
+
+- ``from``: The value to be casted.
+
+**Return value**
+
+- Returns an :ref:`overflow_result <libcudacxx-extended-api-numeric-overflow_result>` object that contains the result of the saturating cast and a boolean indicating whether an overflow occurred.
+
+**Constraints**
+
+- ``To`` and ``From`` must be `integer types <https://eel.is/c++draft/basic.fundamental#1>`_.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/numeric>
+    #include <cuda/std/cassert>
+    #include <cuda/std/limits>
+
+    __global__ void kernel()
+    {
+        constexpr auto int_max = cuda::std::numeric_limits<int>::max();
+        constexpr auto int_min = cuda::std::numeric_limits<int>::min();
+
+        if (auto result = cuda::saturating_overflow_cast<unsigned>(int_max))
+        {
+            assert(false); // Should not be reached
+        }
+        else
+        {
+            assert(result.value == static_cast<unsigned>(int_max));
+        }
+
+        if (auto result = cuda::saturating_overflow_cast<unsigned>(int_min)) // saturated
+        {
+            assert(result.value == 0);
+        }
+        else
+        {
+            assert(false); // Should not be reached
+        }
+    }
+
+    int main()
+    {
+        kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/jcv39ohso>`_

cccl_upstream/docs/libcudacxx/extended_api/numeric/saturating_sub_overflow.rst

@@ -0,0 +1,78 @@
+.. _libcudacxx-extended-api-numeric-saturating_sub_overflow:
+
+``cuda::saturating_sub_overflow``
+=================================
+
+Defined in ``<cuda/numeric>`` header.
+
+.. code:: cpp
+
+   namespace cuda {
+
+   template <class T>
+   struct overflow_result;
+
+   template <class T>
+   [[nodiscard]] __host__ __device__ constexpr
+   overflow_result<T> saturating_sub_overflow(T lhs, T rhs) noexcept; // (1)
+
+   template <class T>
+   [[nodiscard]] __host__ __device__ constexpr
+   bool saturating_sub_overflow(T& result, T lhs, T rhs) noexcept; // (2)
+
+   } // namespace cuda
+
+The function ``cuda::saturating_sub_overflow`` performs saturating subtraction of two values ``lhs`` and ``rhs`` with overflow detection.
+
+**Parameters**
+
+- ``result``: The result of the saturating subtraction. (2)
+- ``lhs``: The left-hand side operand. (1, 2)
+- ``rhs``: The right-hand side operand. (1, 2)
+
+**Return value**
+
+1. Returns an :ref:`overflow_result <libcudacxx-extended-api-numeric-overflow_result>` object containing the result of the saturating subtraction and a boolean flag indicating whether an overflow or underflow occurred.
+2. Returns ``true`` if an overflow or underflow occurred, ``false`` otherwise.
+
+**Constraints**
+
+- ``T`` must be an `integer type <https://eel.is/c++draft/basic.fundamental#1>`_.
+
+**Performance considerations**
+
+- Functionality is implemented by correcting the ``cuda::sub_overflow`` result in case of overflow/underflow.
+- Unsigned computations are generally faster than signed computations.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/numeric>
+    #include <cuda/std/cassert>
+    #include <cuda/std/limits>
+
+    __global__ void kernel()
+    {
+        constexpr auto uint_max = cuda::std::numeric_limits<unsigned>::max();
+
+        const auto result = cuda::saturating_sub_overflow(42u, uint_max); // saturated
+        assert(result.value == 0);
+        assert(result.overflow);
+
+        int value;
+        if (cuda::saturating_sub_overflow(value, 42, 1024))
+        {
+            assert(false); // shouldn't be reached
+        }
+        assert(value == -982);
+    }
+
+    int main()
+    {
+        kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/1M9j9oG9z>`_

cccl_upstream/docs/libcudacxx/extended_api/numeric/sub_overflow.rst

@@ -0,0 +1,87 @@
+.. _libcudacxx-extended-api-numeric-sub_overflow:
+
+``cuda::sub_overflow``
+======================
+
+Defined in ``<cuda/numeric>`` header.
+
+.. code:: cpp
+
+   namespace cuda {
+
+   template <class T>
+   struct overflow_result;
+
+   template <class Result = /*unspecified*/, class Lhs, class Rhs>
+   [[nodiscard]] __host__ __device__ constexpr
+   overflow_result</*see-below*/> sub_overflow(Lhs lhs, Rhs rhs) noexcept; // (1)
+
+   template <class Result, class Lhs, class Rhs>
+   [[nodiscard]] __host__ __device__ constexpr
+   bool sub_overflow(Result& result, Lhs lhs, Rhs rhs) noexcept; // (2)
+
+   } // namespace cuda
+
+The function ``cuda::sub_overflow`` performs subtraction of two values ``lhs`` and ``rhs`` with overflow detection. The result is the same as if the operands were first promoted to an infinite precision signed type, subtracted, and then the result truncated to the type of the return value.
+
+**Parameters**
+
+- ``result``: The result of the subtraction (2).
+- ``lhs``: The left-hand side operand (1, 2).
+- ``rhs``: The right-hand side operand (1, 2).
+
+**Return value**
+
+1. Returns an :ref:`overflow_result <libcudacxx-extended-api-numeric-overflow_result>` object containing the computed  difference and a boolean flag indicating whether an overflow or underflow occurred. If the ``Result`` type is specified, it will be used as the type of the result, otherwise the common type of ``Lhs`` and ``Rhs`` is used.
+2. Returns ``true`` if an overflow or underflow occurred, ``false`` otherwise.
+
+**Constraints**
+
+- ``Result``, ``Lhs``, and ``Rhs`` must be `integer types <https://eel.is/c++draft/basic.fundamental#1>`_.
+
+**Performance considerations**
+
+- No overflow checking is required when ``lhs - rhs`` is always representable with the ``Result`` type.
+- Computation is generally faster when ``Lhs``, ``Rhs``, and ``Result`` have the same signedness.
+- Unsigned computations are generally faster than signed computations.
+- The function uses PTX ``asm`` on device and compiler intrinsics on host whenever possible.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/numeric>
+    #include <cuda/std/cassert>
+    #include <cuda/std/limits>
+
+    __global__ void kernel()
+    {
+        constexpr auto int_max = cuda::std::numeric_limits<int>::max();
+        constexpr auto int_min = cuda::std::numeric_limits<int>::min();
+
+        // cuda::sub_overflow(lhs, rhs) returning the common type of lhs and rhs
+        auto result = cuda::sub_overflow(3, 1))
+        assert(result.value == 2);
+
+
+        // cuda::sub_overflow<Result>(lhs, rhs) with an explicit result type
+        auto result2 = cuda::sub_overflow<long long>(int_min, 1);
+        assert(!result2.overflow);
+        assert(result2.value == static_cast<long long>(int_min) - 1ll);
+
+        unsigned result3{};
+        // cuda::sub_overflow(result, lhs, rhs) returning a bool flag
+        if (!cuda::sub_overflow(result, 5u, 3u))
+        {
+            assert(result == 2u);
+        }
+    }
+
+    int main()
+    {
+        kernel<<<1, 1>>>();
+        cudaDeviceSynchronize();
+    }
+
+`See it on Godbolt 🔗 <https://godbolt.org/z/Pq8sc9s7a>`_

cccl_upstream/docs/libcudacxx/extended_api/random.rst

@@ -0,0 +1,24 @@
+.. _libcudacxx-extended-api-random:
+
+Random
+======
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   random/pcg64
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`cuda::pcg64 <libcudacxx-extended-api-random-pcg64>`
+     - 128-bit state PCG engine with 64-bit output
+     - CCCL 3.3.0
+     - CUDA 13.3

cccl_upstream/docs/libcudacxx/extended_api/random/pcg64.rst

@@ -0,0 +1,23 @@
+.. _libcudacxx-extended-api-random-pcg64:
+
+``pcg64``
+=========
+
+Defined in the ``<cuda/random>`` header.
+
+``cuda::pcg64`` is a 128-bit state PCG XSL RR 128/64 engine that produces 64-bit unsigned integer outputs. It has a
+period of ``2^128`` and supports logarithmic-time ``discard``. ``cuda::pcg64`` models the
+`UniformRandomBitGenerator <https://en.cppreference.com/w/cpp/named_req/UniformRandomBitGenerator>`_ named requirement.
+
+Example
+-------
+
+.. code:: cuda
+
+    #include <cuda/random>
+
+    __global__ void sample_kernel() {
+        cuda::pcg64 rng(42);
+        auto value = rng();
+        rng.discard(10);
+    }

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives.rst

@@ -0,0 +1,137 @@
+.. _libcudacxx-extended-api-synchronization:
+
+Synchronization Primitives
+===========================
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   synchronization_primitives/atomic
+   synchronization_primitives/atomic_ref
+   synchronization_primitives/latch
+   synchronization_primitives/barrier
+   synchronization_primitives/counting_semaphore
+   synchronization_primitives/binary_semaphore
+   synchronization_primitives/pipeline
+
+.. rubric:: Atomics
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`cuda::atomic <libcudacxx-extended-api-synchronization-atomic>`
+     - System-wide `std::atomic <https://en.cppreference.com/w/cpp/atomic/atomic>`_ objects and operations
+     - libcu++ 1.0.0 / CCCL 2.0.0
+     - CUDA 10.2
+
+   * - :ref:`cuda::atomic_ref <libcudacxx-extended-api-synchronization-atomic-ref>`
+     - System-wide `std::atomic_ref <https://en.cppreference.com/w/cpp/atomic/atomic_ref>`_ objects and operations
+     - libcu++ 1.7.0 / CCCL 2.0.0
+     - CUDA 11.6
+
+.. rubric:: Latches
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`cuda::latch <libcudacxx-extended-api-synchronization-latch>`
+     - System-wide `std::latch <https://en.cppreference.com/w/cpp/thread/latch>`_ single-phase asynchronous
+       thread coordination mechanism
+     - libcu++ 1.1.0 / CCCL 2.0.0
+     - CUDA 11.0
+
+.. rubric:: Barriers
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`cuda::barrier <libcudacxx-extended-api-synchronization-barrier>`
+     - System wide `std::barrier <https://en.cppreference.com/w/cpp/thread/barrier>`_ multi-phase asynchronous
+       thread coordination mechanism
+     - libcu++ 1.1.0 / CCCL 2.0.0
+     - CUDA 11.0
+
+.. rubric:: Semaphores
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`cuda::counting_semaphore <libcudacxx-extended-api-synchronization-counting-semaphore>`
+     - System wide `std::counting_semaphore <https://en.cppreference.com/w/cpp/thread/counting_semaphore>`_
+       primitive for constraining concurrent access
+     - libcu++ 1.1.0 / CCCL 2.0.0
+     - CUDA 11.0
+
+   * - :ref:`cuda::binary_semaphore <libcudacxx-extended-api-synchronization-counting-semaphore>`
+     - System wide `std::binary_semaphore <https://en.cppreference.com/w/cpp/thread/counting_semaphore>`_
+       primitive for mutual exclusion
+     - libcu++ 1.1.0 / CCCL 2.0.0
+     - CUDA 11.0
+
+.. rubric:: Pipelines
+
+The pipeline library is included in the CUDA Toolkit, but is not part of the open source libcu++ distribution.
+
+.. list-table::
+   :widths: 25 45 30 30
+   :header-rows: 1
+
+   * - **Header**
+     - **Content**
+     - **CCCL Availability**
+     - **CUDA Toolkit Availability**
+
+   * - :ref:`cuda::pipeline <libcudacxx-extended-api-synchronization-pipeline>`
+     - Coordination mechanism for sequencing asynchronous operations
+     - libcu++ 1.2.0 / CCCL 2.0.0
+     - CUDA 11.1
+
+   * - :ref:`cuda::pipeline_shared_state <libcudacxx-extended-api-synchronization-pipeline-pipeline-shared-state>`
+     - :ref:`cuda::pipeline <libcudacxx-extended-api-synchronization-pipeline>` shared state object
+     - libcu++ 1.1.0 / CCCL 2.0.0
+     - CUDA 11.0
+
+   * - :ref:`cuda::pipeline_role <libcudacxx-extended-api-synchronization-pipeline-pipeline-role>`
+     - Defines producer/consumer role for a thread participating in a *pipeline*
+     - libcu++ 1.1.0 / CCCL 2.0.0
+     - CUDA 11.0
+
+   * - :ref:`cuda::make_pipeline <libcudacxx-extended-api-synchronization-pipeline-pipeline-role>`
+     - Creates a :ref:`cuda::pipeline <libcudacxx-extended-api-synchronization-pipeline>`
+     - libcu++ 1.1.0 / CCCL 2.0.0
+     - CUDA 11.0
+
+   * - :ref:`cuda::pipeline_consumer_wait_prior <libcudacxx-extended-api-synchronization-pipeline-pipeline-consumer-wait-prior>`
+     - Blocks the current thread until all operations committed up to a prior *pipeline stage* complete
+     - libcu++ 1.1.0 / CCCL 2.0.0
+     - CUDA 11.0
+
+   * - :ref:`cuda::pipeline_producer_commit <libcudacxx-extended-api-synchronization-pipeline-pipeline-producer-commit>`
+     - Binds operations previously issued by the current thread to a :ref:`cuda::barrier <libcudacxx-extended-api-synchronization-barrier>`
+     - libcu++ 1.1.0 / CCCL 2.0.0
+     - CUDA 11.0

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/atomic.rst

@@ -0,0 +1,99 @@
+.. _libcudacxx-extended-api-synchronization-atomic:
+
+``cuda::atomic``
+================
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   atomic/atomic_thread_fence
+   atomic/fetch_max
+   atomic/fetch_min
+
+Defined in header ``<cuda/atomic>``:
+
+.. code:: cuda
+
+   template <typename T, cuda::thread_scope Scope = cuda::thread_scope_system>
+   class cuda::atomic;
+
+The class template ``cuda::atomic`` is an extended form of `cuda::std::atomic <https://en.cppreference.com/w/cpp/atomic/atomic>`_
+that takes an additional :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` argument,
+defaulted to ``cuda::std::thread_scope_system``.
+
+It has the same interface and semantics as `cuda::std::atomic <https://en.cppreference.com/w/cpp/atomic/atomic>`_,
+with the following additional operations.
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - :ref:`cuda::atomic_thread_fence <libcudacxx-extended-api-synchronization-atomic-atomic-thread-fence>`
+     - Memory order and scope dependent fence synchronization primitive.
+   * - :ref:`cuda::atomic::fetch_min <libcudacxx-extended-api-synchronization-atomic-atomic-fetch-min>`
+     - Atomically find the minimum of the stored value and a provided value.
+   * - :ref:`cuda::atomic::fetch_max <libcudacxx-extended-api-synchronization-atomic-atomic-fetch-max>`
+     - Atomically find the maximum of the stored value and a provided value.
+
+Concurrency Restrictions
+------------------------
+
+An object of type ``cuda::atomic`` or `cuda::std::atomic <https://en.cppreference.com/w/cpp/atomic/atomic>`_
+shall not be accessed concurrently by CPU and GPU threads unless:
+
+  - it is in unified memory and the `concurrentManagedAccess property <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaDeviceProp.html#structcudaDeviceProp_116f9619ccc85e93bc456b8c69c80e78b>`_
+    is 1, or
+  - it is in CPU memory and the `hostNativeAtomicSupported property <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaDeviceProp.html#structcudaDeviceProp_1ef82fd7d1d0413c7d6f33287e5b6306f>`_
+    is 1.
+
+Note, for objects of scopes other than ``cuda::thread_scope_system`` this is a data-race, and therefore also
+prohibited regardless of memory characteristics.
+
+Under CUDA Compute Capability 6 (Pascal), an object of type ``atomic`` may not be used:
+
+  - with automatic storage duration, or
+  - if ``is_always_lock_free()`` is ``false``.
+
+Under CUDA Compute Capability prior to 6 (Pascal), objects of type ``cuda::atomic`` or
+`cuda::std::atomic <https://en.cppreference.com/w/cpp/atomic/atomic>`_ may not be used.
+
+Implementation-Defined Behavior
+-------------------------------
+
+For each type ``T`` and :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` ``S``,
+the value of ``cuda::atomic<T, S>::is_always_lock_free()`` is as follows:
+
+.. list-table::
+   :widths: 25 25 50
+   :header-rows: 0
+
+   * - Type ``T``
+     - :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` ``S``
+     - ``cuda::atomic<T, S>::is_always_lock_free()``
+   * - Any valid type
+     - Any thread scope
+     - ``sizeof(T) <= 8``
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/atomic>
+
+   __global__ void example_kernel() {
+     // This atomic is suitable for all threads in the system.
+     cuda::atomic<int, cuda::thread_scope_system> a;
+
+     // This atomic has the same type as the previous one (`a`).
+     cuda::atomic<int> b;
+
+     // This atomic is suitable for all threads on the current processor (e.g. GPU).
+     cuda::atomic<int, cuda::thread_scope_device> c;
+
+     // This atomic is suitable for threads in the same thread block.
+     cuda::atomic<int, cuda::thread_scope_block> d;
+   }
+
+`See it on Godbolt <https://godbolt.org/z/avo3Evbee>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/atomic/atomic_thread_fence.rst

@@ -0,0 +1,51 @@
+.. _libcudacxx-extended-api-synchronization-atomic-atomic-thread-fence:
+
+cuda::atomic::atomic_thread_fence
+=====================================
+
+Defined in header ``<cuda/atomic>``:
+
+.. code:: cuda
+
+   __host__ __device__
+   void cuda::atomic_thread_fence(cuda::std::memory_order order,
+                                  cuda::thread_scope scope = cuda::thread_scope_system);
+
+Establishes memory synchronization ordering of non-atomic and relaxed atomic accesses, as instructed by ``order``,
+for all threads within ``scope`` without an associated atomic operation. It has the same semantics as
+`cuda::std::atomic_thread_fence <https://en.cppreference.com/w/cpp/atomic/atomic_thread_fence>`_.
+
+Example
+-------
+
+The following code is an example of the :ref:`MessagePassing <libcudacxx-extended-api-memory-model-message-passing>` pattern:
+
+.. code:: cuda
+
+   #include <cstdio>
+   #include <cuda/atomic>
+   #include <cooperative_groups.h>
+
+   namespace cg = cooperative_groups;
+
+   __global__ void example_kernel(int* data, cuda::std::atomic_flag* flag) {
+     assert(cg::grid_group::size() == 2);
+     assert(cg::thread_block::size() == 1);
+
+     if (blockIdx.x == 0) {
+       *data = 42;
+       cuda::atomic_thread_fence(cuda::memory_order_release,
+                                 cuda::thread_scope_device);
+       flag->test_and_set(cuda::std::memory_order_relaxed);
+       flag->notify_one();
+     }
+     else {
+       // an atomic operation is required to set up the synchronization
+       flag->wait(false, cuda::std::memory_order_relaxed);
+       cuda::atomic_thread_fence(cuda::memory_order_acquire,
+                                 cuda::thread_scope_device);
+       std::printf("%d\n", *data); // Prints 42
+     }
+   }
+
+`See it on Godbolt <https://godbolt.org/z/aG37o5qxx>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/atomic/fetch_max.rst

@@ -0,0 +1,34 @@
+.. _libcudacxx-extended-api-synchronization-atomic-atomic-fetch-max:
+
+cuda::atomic::fetch_max
+===========================
+
+Defined in header ``<cuda/atomic>``:
+
+.. code:: cuda
+
+   template <typename T, cuda::thread_scope Scope>
+   __host__ __device__
+   T cuda::atomic<T, Scope>::fetch_max(T const& val,
+                                       cuda::std::memory_order order
+                                         = cuda::std::memory_order_seq_cst);
+
+Atomically find the maximum of the value stored in the ``cuda::atomic``
+and ``val``. The maximum is found using
+`cuda::std::max <https://en.cppreference.com/w/cpp/algorithm/max>`_.
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/atomic>
+
+   __global__ void example_kernel() {
+     cuda::atomic<int> a(0);
+     auto x = a.fetch_max(1);
+     auto y = a.load();
+     assert(x == 0 && y == 1);
+   }
+
+`See it on Godbolt <https://godbolt.org/z/rexn5T78G>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/atomic/fetch_min.rst

@@ -0,0 +1,34 @@
+.. _libcudacxx-extended-api-synchronization-atomic-atomic-fetch-min:
+
+cuda::atomic::fetch_min
+===========================
+
+Defined in header ``<cuda/atomic>``:
+
+.. code:: cuda
+
+   template <typename T, cuda::thread_scope Scope>
+   __host__ __device__
+   T cuda::atomic<T, Scope>::fetch_min(T const& val,
+                                       cuda::std::memory_order order
+                                         = cuda::std::memory_order_seq_cst);
+
+Atomically find the minimum of the value stored in the ``cuda::atomic``
+and ``val``. The minimum is found using
+`cuda::std::min <https://en.cppreference.com/w/cpp/algorithm/min>`_.
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/atomic>
+
+   __global__ void example_kernel() {
+     cuda::atomic<int> a(1);
+     auto x = a.fetch_min(0);
+     auto y = a.load();
+     assert(x == 1 && y == 0);
+   }
+
+`See it on Godbolt <https://godbolt.org/z/vMj9e5hdv>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/atomic_ref.rst

@@ -0,0 +1,108 @@
+.. _libcudacxx-extended-api-synchronization-atomic-ref:
+
+``cuda::atomic_ref``
+====================
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+
+Defined in header ``<cuda/atomic>``:
+
+.. code:: cuda
+
+   template <typename T, cuda::thread_scope Scope = cuda::thread_scope_system>
+   class cuda::atomic_ref;
+
+The class template ``cuda::atomic_ref`` is an extended form of `cuda::std::atomic_ref <https://en.cppreference.com/w/cpp/atomic/atomic_ref>`_
+that takes an additional :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` argument, defaulted to
+``cuda::std::thread_scope_system``.
+
+It has the same interface and semantics as `cuda::std::atomic_ref <https://en.cppreference.com/w/cpp/atomic/atomic_ref>`_,
+with the following additional operations. This class additionally deviates from the standard by being backported to C++11.
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - :ref:`cuda::atomic_ref::fetch_min <libcudacxx-extended-api-synchronization-atomic-atomic-fetch-min>`
+     - Atomically find the minimum of the stored value and a provided value.
+   * - :ref:`cuda::atomic_ref::fetch_max <libcudacxx-extended-api-synchronization-atomic-atomic-fetch-max>`
+     - Atomically find the maximum of the stored value and a provided value.
+
+
+Limitations
+-----------
+
+``cuda::atomic_ref<T>`` and ``cuda::std::atomic_ref<T>`` may only be instantiated when ``T`` satisfies ``sizeof(T) <= 8`` or ``sizeof(T) <= 16`` when requirements are met.
+
+The operations available to ``T`` when ``sizeof(T) == 16`` depend on the architecture:
+  - On SM70 and later: ``load`` and ``store`` are supported.
+  - On SM90 and later: ``fetch_*`` and synchronization operations are supported, implemented via atomic compare-and-swap (CAS).
+
+No object or subobject of an object referenced by an ``atomic_­ref`` shall be concurrently referenced by any other
+``atomic_­ref`` that has a different ``Scope``.
+
+For ``cuda::atomic_ref<T>`` and ``cuda::std::atomic_ref<T>`` the type ``T`` must satisfy the following:
+  - ``sizeof(T) <= 16``.
+  - The referenced object must be aligned to its size: ``alignof(T) == sizeof(T)``.
+  - ``T`` must not have "padding bits", i.e., T's `object representation <https://en.cppreference.com/w/cpp/language/object#Object_representation_and_value_representation>`_
+    must not have bits that do not participate in it's value representation.
+
+Concurrency Restrictions
+------------------------
+
+See :ref:`memory model <libcudacxx-extended-api-memory-model>` documentation for general restrictions on atomicity.
+
+With CUDA Compute Capability 6 (Pascal), an object of type ``atomic_ref`` may not be used:
+  - with a reference to an object with a automatic storage duration in a GPU thread, or
+  - if ``is_always_lock_free()`` is ``false``.
+
+For CUDA Compute Capability prior to 6 (Pascal), objects of type ``cuda::atomic_ref`` or
+`cuda::std::atomic_ref <https://en.cppreference.com/w/cpp/atomic/atomic_ref>`_ may not be used.
+
+Implementation-Defined Behavior
+-------------------------------
+
+For each type ``T`` and :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` ``S``, the value of
+``cuda::atomic_ref<T, S>::is_always_lock_free()`` and ``cuda::std::atomic_ref<T>::is_always_lock_free()`` is as follows:
+
+.. list-table::
+   :widths: 25 25 50
+   :header-rows: 0
+
+   * - Type ``T``
+     - :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` ``S``
+     - ``cuda::atomic_ref<T, S>::is_always_lock_free()``
+   * - Any valid type
+     - Any thread scope
+     - ``sizeof(T) <= 8``
+
+Types of ``T``, where ``sizeof(T) < 4``, are not natively supported by the underlying hardware. For these types atomic
+operations are emulated and will be drastically slower. Contention with contiguous memory in the current 4 byte boundary
+will be exacerbated. In these situations it is advisable to perform a hierarchical reduction to non-adjacent memory first.
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/atomic>
+
+   __global__ void example_kernel(int *gmem, int *pinned_mem) {
+     // This atomic is suitable for all threads in the system.
+     cuda::atomic_ref<int, cuda::thread_scope_system> a(*pinned_mem);
+
+     // This atomic has the same type as the previous one (`a`).
+     cuda::atomic_ref<int> b(*pinned_mem);
+
+     // This atomic is suitable for all threads on the current processor (e.g. GPU).
+     cuda::atomic_ref<int, cuda::thread_scope_device> c(*gmem);
+
+     __shared__ int shared_v;
+     // This atomic is suitable for threads in the same thread block.
+     cuda::atomic_ref<int, cuda::thread_scope_block> d(shared_v);
+   }
+
+`See it on Godbolt <https://godbolt.org/z/fr4K7ErEh>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/barrier.rst

@@ -0,0 +1,297 @@
+.. _libcudacxx-extended-api-synchronization-barrier:
+
+``cuda::barrier``
+=================
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   barrier/init
+   barrier/barrier_native_handle
+   barrier/barrier_arrive_tx
+   barrier/barrier_expect_tx
+
+Defined in header ``<cuda/barrier>``:
+
+.. code:: cuda
+
+   template <cuda::thread_scope Scope,
+             typename CompletionFunction = /* unspecified */>
+   class cuda::barrier;
+
+The class template ``cuda::barrier`` is an extended form of `cuda::std::barrier <https://en.cppreference.com/w/cpp/thread/barrier>`_
+that takes an additional :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` argument.
+
+If ``!(scope == cuda::thread_block_scope && cuda::device::is_address_from(this, cuda::device::address_space::shared))``, then the semantics are the same as
+`cuda::std::barrier <https://en.cppreference.com/w/cpp/thread/barrier>`_, otherwise, see below.
+
+The ``cuda::barrier`` class templates extends ``cuda::std::barrier`` with the following additional operations:
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - :ref:`cuda::barrier::init <libcudacxx-extended-api-synchronization-barrier-barrier-init>`
+     - Initialize a ``cuda::barrier``.
+   * - :ref:`cuda::device::barrier_native_handle <libcudacxx-extended-api-synchronization-barrier-barrier-native-handle>`
+     - Get the native handle to a ``cuda::barrier``.
+   * - :ref:`cuda::device::barrier_arrive_tx <libcudacxx-extended-api-synchronization-barrier-barrier-arrive-tx>`
+     - Arrive on a ``cuda::barrier<cuda::thread_scope_block>`` with transaction count update.
+   * - :ref:`cuda::device::barrier_expect_tx <libcudacxx-extended-api-synchronization-barrier-barrier-expect-tx>`
+     - Update transaction count of ``cuda::barrier<cuda::thread_scope_block>``.
+
+If ``scope == cuda::thread_scope_block && cuda::device::is_address_from(this, cuda::device::address_space::shared)``, then the semantics of `[thread.barrier.class] <http://eel.is/c++draft/thread.barrier.class>`_
+of ISO/IEC IS 14882 (the C++ Standard) are modified as follows:
+
+   A barrier is a thread coordination mechanism whose lifetime consists
+   of a sequence of barrier phases, where each phase allows at most an
+   expected number of threads to block until the expected number of
+   threads **and the expected number of transaction-based asynchronous
+   operations** arrive at the barrier. Each *barrier phase* consists of
+   the following steps:
+
+   1. The *expected count* is decremented by each call to ``arrive``,\ ``arrive_and_drop``\ **,
+      or cuda::device::barrier_arrive_tx**.
+   2. **The transaction count is incremented by each call to cuda::device::barrier_arrive_tx and decremented by the
+      completion of transaction-based asynchronous operations such as cuda::memcpy_async_tx.**
+   3. Exactly once after **both** the *expected count* **and the transaction count** reach zero, a thread executes the
+      *completion step* during its call to ``arrive``, ``arrive_and_drop``, ``cuda::device::barrier_arrive_tx``,
+      or ``wait``, except that it is implementation-defined whether the step executes if no thread calls ``wait``.
+   4. When the completion step finishes, the *expected count* is reset to what was specified by the ``expected``
+      argument to the constructor, possibly adjusted by calls to ``arrive_and_drop``, and the next phase starts.
+
+   Concurrent invocations of the member functions of barrier **and the non-member barrier APIs in cuda::device**,
+   other than its destructor, do not introduce data races. The member functions ``arrive`` and ``arrive_and_drop``,
+   **and the non-member function cuda::device::barrier_arrive_tx**, execute atomically.
+
+.. rubric:: NVCC ``__shared__`` Initialization Warnings
+
+When using libcu++ with NVCC, a ``__shared__`` ``cuda::barrier`` will lead to the following warning because
+``__shared__`` variables are not initialized:
+
+.. code:: bash
+
+   warning: dynamic initialization is not supported for a function-scope static
+   __shared__ variable within a __device__/__global__ function
+
+It can be silenced using ``#pragma nv_diag_suppress static_var_with_dynamic_init``.
+
+To properly initialize a ``__shared__`` ``cuda::barrier``, use the
+:ref:`cuda::barrier::init <libcudacxx-extended-api-synchronization-barrier-barrier-init>` friend function.
+
+.. rubric:: Concurrency Restrictions
+
+An object of type ``cuda::barrier`` or ``cuda::std::barrier`` shall not be accessed concurrently by CPU and GPU threads unless:
+
+   - it is in unified memory and the `concurrentManagedAccess property <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaDeviceProp.html#structcudaDeviceProp_116f9619ccc85e93bc456b8c69c80e78b>`_
+     is 1, or
+   - it is in CPU memory and the `hostNativeAtomicSupported property <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaDeviceProp.html#structcudaDeviceProp_1ef82fd7d1d0413c7d6f33287e5b6306f>`_
+     is 1.
+
+Note, for objects of scopes other than ``cuda::thread_scope_system`` this is a data-race, and therefore also prohibited
+regardless of memory characteristics.
+
+Under CUDA Compute Capability 8 (Ampere) or above, when an object of type ``cuda::barrier<thread_scope_block>`` is
+placed in ``__shared__`` memory, the member function ``arrive`` performs a reduction of the arrival count among
+`coalesced threads <https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#coalesced-group-cg>`_ followed
+by the arrival operation in one thread. Programs shall ensure that this transformation would not introduce errors,
+for example relative to the requirements of `thread.barrier.class paragraph 12 <https://eel.is/c++draft/thread.barrier.class#12>`_
+of ISO/IEC IS 14882 (the C++ Standard).
+
+Under CUDA Compute Capability 6 (Pascal) or prior, an object of type ``cuda::barrier`` or ``cuda::std::barrier`` may
+not be used.
+
+.. rubric:: Shared memory barriers with transaction count
+
+In addition to the arrival count, a ``cuda::barrier<thread_scope_block>`` object located in shared memory supports a
+`tx-count <https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-tracking-async-operations>`_,
+which is used for tracking the completion of some asynchronous memory operations or transactions.
+The tx-count tracks the number of asynchronous transactions, in units specified by the asynchronous memory operation
+(typically bytes), that are outstanding and yet to be complete. This capability is exposed, starting with the Hopper
+architecture (CUDA Compute Capability 9).
+
+The tx-count of ``cuda::barrier`` must be set to the total amount of asynchronous memory operations, in units as
+specified by the asynchronous operations, to be tracked by the current phase. This can be achieved with the
+:ref:`cuda::device::barrier_arrive_tx <libcudacxx-extended-api-synchronization-barrier-barrier-arrive-tx>` function call.
+
+Upon completion of each of the asynchronous operations, the tx-count of the ``cuda::barrier`` will be updated and thus
+progress the ``cuda::barrier`` towards the completion of the current phase. This may complete the current phase.
+
+.. rubric:: Implementation-Defined Behavior
+
+For each :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` ``S`` and ``CompletionFunction``
+``F``, the value of ``cuda::barrier<S, F>::max()`` is as follows:
+
+.. list-table::
+   :widths: 25 25 50
+   :header-rows: 0
+
+   * - :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` ``S``
+     - ``CompletionFunction`` ``F``
+     - ``barrier<S, F>::max()``
+   * - ``cuda::thread_scope_block``
+     - Default or user-provided
+     - ``(1 << 20) - 1``
+   * - *Not* ``cuda::thread_scope_block``
+     - Default
+     - ``cuda::std::numeric_limits<cuda::std::int32_t>::max()``
+   * - *Not* ``cuda::thread_scope_block``
+     - User-provided
+     - ``cuda::std::numeric_limits<cuda::std::ptrdiff_t>::max()``
+
+.. rubric:: Example
+
+.. code:: cuda
+
+   #include <cuda/barrier>
+
+   __global__ void example_kernel() {
+     // This barrier is suitable for all threads in the system.
+     cuda::barrier<cuda::thread_scope_system> a(10);
+
+     // This barrier has the same type as the previous one (`a`).
+     cuda::std::barrier<> b(10);
+
+     // This barrier is suitable for all threads on the current processor (e.g. GPU).
+     cuda::barrier<cuda::thread_scope_device> c(10);
+
+     // This barrier is suitable for all threads in the same thread block.
+     cuda::barrier<cuda::thread_scope_block> d(10);
+   }
+
+`See it on Godbolt <https://godbolt.org/z/7Kbz5qqhh>`__
+
+.. rubric:: Example: 1D TMA load of two buffers with arrival token (sm90+)
+
+The following example shows how to use TMA to load two tiles of data from global memory into shared memory:
+
+.. code:: cuda
+
+   #include <cuda/barrier>
+   #include <cuda/ptx>
+
+   // selects a single leader thread from the block
+   __device__ bool elect_one() {
+     const unsigned int tid = threadIdx.x;
+     const unsigned int warp_id = tid / 32;
+     const unsigned int uniform_warp_id = __shfl_sync(0xFFFFFFFF, warp_id, 0); // broadcast from lane 0
+     return uniform_warp_id == 0 && cuda::ptx::elect_sync(0xFFFFFFFF); // elect a leader thread among warp 0
+   }
+
+   __global__ void example_kernel(int* gmem1, double* gmem2) {
+     constexpr int tile_size = 1024;
+     __shared__ alignas(16)    int smem1[tile_size];
+     __shared__ alignas(16) double smem2[tile_size];
+
+     #pragma nv_diag_suppress static_var_with_dynamic_init
+     __shared__  cuda::barrier<cuda::thread_scope_block> bar;
+
+     // setup the barrier where each thread in the block arrives at
+     if (threadIdx.x == 0) {
+       init(&bar, blockDim.x);
+     }
+     __syncthreads(); // need to sync so other threads can arrive
+
+     // select a single thread from the block and issue two TMA bulk copy operations
+     const auto elected = elect_one();
+     if (elected) {
+       cuda::device::memcpy_async_tx(smem1, gmem1, cuda::aligned_size_t<16>(tile_size * sizeof(int)   ), bar);
+       cuda::device::memcpy_async_tx(smem2, gmem2, cuda::aligned_size_t<16>(tile_size * sizeof(double)), bar);
+     }
+
+     // arrive at the barrier
+     // the elected thread also updates the barrier's expect_tx with the **total** number of loaded bytes
+     const int tx_count = elected ? tile_size * (sizeof(int) + sizeof(double)) : 0;
+     auto token = cuda::device::barrier_arrive_tx(bar, 1, tx_count);
+
+     // wait for TMA copies to complete
+     bar.wait(cuda::std::move(token));
+
+     // process data in smem ...
+   }
+
+`See it on Godbolt <https://godbolt.org/z/ddhzGeWPE>`__
+
+Data is loaded from the global memory pointers ``gmem1`` and ``gmem2``
+into the shared memory buffers ``smem1`` and ``smem2``.
+The shared memory buffers have to be aligned to 16 bytes.
+Additionally, a single barrier with block scope is setup by a single leader thread.
+Each thread will arrive at the barrier, so ``blockDim.x`` is passed as arrival count to ``init``.
+This barrier is used to synchronize the asynchronous TMA copies with the rest of the threads in the block.
+The leader initiates the TMA copies using ``ptx::cp_async_bulk``,
+and updates the barrier's tx-count with the total number of bytes transferred by the TMA copy operations
+while arriving at the barrier using ``cuda::device::barrier_arrive_tx``.
+All other threads just arrive normally at the barrier.
+All threads then wait on the barrier to complete the current phase by calling ``wait``.
+Once ``wait`` returns, all data is available in shared memory.
+
+.. rubric:: Example: 1D TMA load of two buffers with barrier phase parity check (sm90+)
+
+.. code:: cuda
+
+   #include <cuda/barrier>
+   #include <cuda/ptx>
+
+   // selects a single leader thread from the block
+   __device__ bool elect_one() {
+     const unsigned int tid = threadIdx.x;
+     const unsigned int warp_id = tid / 32;
+     const unsigned int uniform_warp_id = __shfl_sync(0xFFFFFFFF, warp_id, 0); // broadcast from lane 0
+     return uniform_warp_id == 0 && cuda::ptx::elect_sync(0xFFFFFFFF); // elect a leader thread among warp 0
+   }
+
+   __global__ void example_kernel(int* gmem1, double* gmem2) {
+     constexpr int tile_size = 1024;
+     __shared__ alignas(16)    int smem1[tile_size];
+     __shared__ alignas(16) double smem2[tile_size];
+
+     #pragma nv_diag_suppress static_var_with_dynamic_init
+     __shared__  cuda::barrier<cuda::thread_scope_block> bar;
+
+     // setup the barrier where only the leader thread arrives
+     if (elect_one()) {
+       init(&bar, 1);
+
+       // issue two TMA bulk copy operations
+       cuda::device::memcpy_async_tx(smem1, gmem1, cuda::aligned_size_t<16>(tile_size * sizeof(int)   ), bar);
+       cuda::device::memcpy_async_tx(smem2, gmem2, cuda::aligned_size_t<16>(tile_size * sizeof(double)), bar);
+
+       // arrive and update the barrier's expect_tx with the **total** number of loaded bytes
+       (void)cuda::device::barrier_arrive_tx(bar, 1, tile_size * (sizeof(int) + sizeof(double)));
+     }
+     __syncthreads(); // need to sync so the barrier is set up when the other threads arrive and wait
+
+     // wait for the current barrier phase to complete
+     bar.wait_parity(0);
+
+     // process data in smem ...
+   }
+
+`See it on Godbolt <https://godbolt.org/z/oq85PoKKj>`__
+
+This is similar to the above example, but this time only the leader thread arrives at the barrier.
+This has the advantage that only one leader election is necessary
+and a single uniform datapath section is generated.
+This generally generates less instructions.
+Because now we don't get an arrival token in each thread, we cannot use ``wait(token)`` with all threads.
+Instead, we just wait until the end of the barrier's current phase using ``wait_parity(0)``
+(0 is the parity of the current phase).
+
+Before CCCL 3.2, ``bar.wait_parity(0);`` contained additional logic which may have lead to unnecessary instructions.
+If you are using CCCL below 3.2, you may replace this line with:
+
+.. code:: cuda
+
+   while (!cuda::ptx::mbarrier_try_wait_parity(cuda::device::barrier_native_handle(bar), 0))
+     ;
+
+.. rubric:: Example: 1D TMA load and store using `cuda::device::memcpy_async_tx` (sm90+)
+
+This example can be found in :ref:`libcudacxx-extended-api-asynchronous-operations-memcpy-async-tx-example`.
+
+
+.. rubric:: Example: 1D TMA load and store using `cuda::memcpy_async` (sm90+)
+
+This example can be found in the
+`CUDA programming guide <https://docs.nvidia.com/cuda/cuda-c-programming-guide/#using-tma-to-transfer-one-dimensional-arrays>`__.

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/barrier/barrier_arrive_tx.rst

@@ -0,0 +1,77 @@
+.. _libcudacxx-extended-api-synchronization-barrier-barrier-arrive-tx:
+
+cuda::device::barrier_arrive_tx
+===================================
+
+Defined in header ``<cuda/barrier>``:
+
+.. code:: cuda
+
+   __device__
+   cuda::barrier<cuda::thread_scope_block>::arrival_token
+   cuda::device::barrier_arrive_tx(
+     cuda::barrier<cuda::thread_scope_block>& bar,
+     ptrdiff_t arrive_count_update,
+     ptrdiff_t transaction_count_update);
+
+Arrives at a barrier in shared memory, decrementing the arrival count and incrementing the expected transaction count.
+
+Preconditions
+-------------
+
+-  ``cuda::device::is_object_from(bar, cuda::device::address_space::shared) == true``
+-  ``1 <= arrive_count_update && transaction_count_update <= (1 << 20) - 1``
+-  ``0 <= transaction_count_update && transaction_count_update <= (1 << 20) - 1``
+
+Effects
+-------
+
+-  This function constructs an arrival_token object associated with the
+   phase synchronization point for the current phase. Then, decrements
+   the arrival count by ``arrive_count_update`` and increments the
+   expected transaction count by ``transaction_count_update``.
+-  This function executes atomically. The call to this function strongly
+   happens-before the start of the phase completion step for the current
+   phase.
+
+Notes
+-----
+
+This function can only be used under CUDA Compute Capability 9.0 (Hopper) or higher.
+
+To check if ``cuda::device::barrier_arrive_tx`` is available, use the ``__cccl_lib_local_barrier_arrive_tx``
+feature flag, as shown in the example code below.
+
+Return Value
+------------
+
+``cuda::device::barrier_arrive_tx`` returns the constructed ``arrival_token`` object.
+
+Example
+-------
+
+Below example shows only ``cuda::device::barrier_arrive_tx``. A more extensive example can be found in the
+:ref:`cuda::device::memcpy_async_tx <libcudacxx-extended-api-asynchronous-operations-memcpy-async-tx>` documentation.
+
+.. code:: cuda
+
+   #include <cuda/barrier>
+   #include <cuda/std/utility> // cuda::std::move
+
+   #ifndef  __cccl_lib_local_barrier_arrive_tx
+   static_assert(false, "Insufficient libcu++ version: cuda::device::arrive_tx is not yet available.");
+   #endif // __cccl_lib_local_barrier_arrive_tx
+
+   __global__ void example_kernel() {
+     __shared__ cuda::barrier<cuda::thread_scope_block> bar;
+     if (threadIdx.x == 0) {
+       init(&bar, blockDim.x);
+     }
+     __syncthreads();
+
+     auto token = cuda::device::barrier_arrive_tx(bar, 1, 0);
+
+     bar.wait(cuda::std::move(token));
+   }
+
+`See it on Godbolt <https://godbolt.org/z/1vxcGrT8j>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/barrier/barrier_expect_tx.rst

@@ -0,0 +1,70 @@
+.. _libcudacxx-extended-api-synchronization-barrier-barrier-expect-tx:
+
+cuda::device::barrier_expect_tx
+===================================
+
+Defined in header ``<cuda/barrier>``:
+
+.. code:: cuda
+
+   __device__
+   void cuda::device::barrier_expect_tx(
+     cuda::barrier<cuda::thread_scope_block>& bar,
+     ptrdiff_t transaction_count_update);
+
+Increments the expected transaction count of a barrier in shared memory.
+
+Preconditions
+-------------
+
+-  ``cuda::device::is_object_from(bar, cuda::device::address_space::shared) == true``
+-  ``0 <= transaction_count_update && transaction_count_update <= (1 << 20) - 1``
+
+Effects
+-------
+
+-  This function increments the expected transaction count by transaction_count_update``.
+-  This function executes atomically.
+
+Notes
+-----
+
+This function can only be used under CUDA Compute Capability 9.0 (Hopper) or higher.
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/barrier>
+   #include <cuda/std/utility> // cuda::std::move
+
+   #if defined(__CUDA_MINIMUM_ARCH__) && __CUDA_MINIMUM_ARCH__ < 900
+   static_assert(false, "Insufficient CUDA Compute Capability: cuda::device::memcpy_expect_tx is not available.");
+   #endif // __CUDA_MINIMUM_ARCH__
+
+   __device__ alignas(16) int gmem_x[2048];
+
+   __global__ void example_kernel() {
+       using barrier_t = cuda::barrier<cuda::thread_scope_block>;
+     alignas(16) __shared__ int smem_x[1024];
+     __shared__ barrier_t bar;
+
+     if (threadIdx.x == 0) {
+       init(&bar, blockDim.x);
+     }
+     __syncthreads();
+
+     if (threadIdx.x == 0) {
+       cuda::device::memcpy_async_tx(smem_x, gmem_x, cuda::aligned_size_t<16>(sizeof(smem_x)), bar);
+       cuda::device::barrier_expect_tx(bar, sizeof(smem_x));
+     }
+     auto token = bar.arrive(1);
+
+     bar.wait(cuda::std::move(token));
+
+     // smem_x contains the contents of gmem_x[0], ..., gmem_x[1023]
+     smem_x[threadIdx.x] += 1;
+   }
+
+`See it on Godbolt <https://godbolt.org/z/9Yj89P76z>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/barrier/barrier_native_handle.rst

@@ -0,0 +1,46 @@
+.. _libcudacxx-extended-api-synchronization-barrier-barrier-native-handle:
+
+cuda::device::barrier_native_handle
+=======================================
+
+Defined in header ``<cuda/barrier>``:
+
+.. code:: cuda
+
+   __device__ cuda::std::uint64_t* cuda::device::barrier_native_handle(
+     cuda::barrier<cuda::thread_scope_block>& bar);
+
+Returns a pointer to the native handle of a :ref:`cuda::barrier <libcudacxx-extended-api-synchronization-barrier>`
+if its scope is ``cuda::thread_scope_block`` and it is allocated in shared memory.
+The pointer is suitable for use with PTX instructions.
+
+Notes
+-----
+
+If ``bar`` is not in ``__shared__`` memory, the behavior is undefined.
+
+Return Value
+------------
+
+A pointer to the PTX "mbarrier" subobject of the :ref:`cuda::barrier <libcudacxx-extended-api-synchronization-barrier>`
+object.
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/barrier>
+
+   __global__ void example_kernel(cuda::barrier<cuda::thread_scope_block>& bar) {
+     auto ptr = cuda::device::barrier_native_handle(bar);
+
+     asm volatile (
+         "mbarrier.arrive.b64 _, [%0];"
+         :
+         : "l" (ptr)
+         : "memory");
+     // Equivalent to: `(void)b.arrive()`.
+   }
+
+`See it on Godbolt <https://godbolt.org/z/dr4798Y76>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/barrier/init.rst

@@ -0,0 +1,53 @@
+.. _libcudacxx-extended-api-synchronization-barrier-barrier-init:
+
+cuda::barrier::init
+=======================
+
+Defined in header ``<cuda/barrier>``:
+
+.. code:: cuda
+
+   template <cuda::thread_scope Scope,
+             typename CompletionFunction = /* unspecified */>
+   class barrier {
+   public:
+     // ...
+
+     __host__ __device__
+     friend void init(cuda::std::barrier* bar,
+                      cuda::std::ptrdiff_t expected,
+                      CompletionFunction cf = CompletionFunction{});
+   };
+
+The friend function ``cuda::barrier::init`` may be used to initialize an
+:ref:`cuda::barrier <libcudacxx-extended-api-synchronization-barrier>` that has not been initialized.
+
+When using libcu++ with NVCC, ``__shared__`` ``cuda::barrier`` will not have its constructors run because ``__shared__``
+variables are not initialized. ``cuda::barrier::init`` should be used to properly initialize such a
+:ref:`cuda::barrier <libcudacxx-extended-api-synchronization-barrier>`.
+
+An NVCC diagnostic warning about the ignored constructor will be emitted:
+
+.. code:: bash
+
+   warning: dynamic initialization is not supported for a function-scope static
+   __shared__ variable within a __device__/__global__ function
+
+It can be silenced using ``#pragma nv_diag_suppress static_var_with_dynamic_init``.
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/barrier>
+
+   // Disables `cuda::barrier` initialization warning.
+   #pragma nv_diag_suppress static_var_with_dynamic_init
+
+   __global__ void example_kernel() {
+     __shared__ cuda::barrier<cuda::thread_scope_block> bar;
+     init(&bar, 1);
+   }
+
+`See it on Godbolt <https://godbolt.org/z/nK5q3xh34>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/binary_semaphore.rst

@@ -0,0 +1,76 @@
+.. _libcudacxx-extended-api-synchronization-binary-semaphore:
+
+``cuda::binary_semaphore``
+==========================
+
+Defined in header ``<cuda/semaphore>``:
+
+.. code:: cpp
+
+   namespace cuda {
+
+   template <cuda::thread_scope Scope>
+   using binary_semaphore = cuda::std::counting_semaphore<Scope, 1>;
+
+   }
+
+The class template ``cuda::binary_semaphore`` is an extended form of `cuda::std::binary_semaphore <https://en.cppreference.com/w/cpp/thread/counting_semaphore>`_
+that takes an additional :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` argument.
+``cuda::binary_semaphore`` has the same interface and semantics as
+`cuda::std::binary_semaphore <https://en.cppreference.com/w/cpp/thread/counting_semaphore>`_, but
+``cuda::binary_semaphore`` is a class template.
+
+Concurrency Restrictions
+------------------------
+
+An object of type ``cuda::binary_semaphore`` or ``cuda::std::binary_semaphore``, shall not be accessed concurrently by
+CPU and GPU threads unless:
+
+   - it is in unified memory and the `concurrentManagedAccess property <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaDeviceProp.html#structcudaDeviceProp_116f9619ccc85e93bc456b8c69c80e78b>`_
+     is 1, or
+   - it is in CPU memory and the `hostNativeAtomicSupported property <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaDeviceProp.html#structcudaDeviceProp_1ef82fd7d1d0413c7d6f33287e5b6306f>`_
+     is 1.
+
+Note, for objects of scopes other than ``cuda::thread_scope_system`` this is a data-race, and therefore also prohibited
+regardless of memory characteristics.
+
+Under CUDA Compute Capability 6 (Pascal) or prior, an object of type ``cuda::binary_semaphore`` or
+``cuda::std::binary_semaphore`` may not be used.
+
+Implementation-Defined Behavior
+-------------------------------
+
+For each :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` ``S``,
+``cuda::binary_semaphore<S>::max()`` is as follows:
+
+.. list-table::
+   :widths: 50 50
+   :header-rows: 0
+
+   * - :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` ``S``
+     - ``cuda::binary_semaphore<S>::max()``
+   * - Any thread scope
+     - ``1``
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/semaphore>
+
+   __global__ void example_kernel() {
+     // This semaphore is suitable for all threads in the system.
+     cuda::binary_semaphore<cuda::thread_scope_system> a;
+
+     // This semaphore has the same type as the previous one (`a`).
+     cuda::std::binary_semaphore<> b;
+
+     // This semaphore is suitable for all threads on the current processor (e.g. GPU).
+     cuda::binary_semaphore<cuda::thread_scope_device> c;
+
+     // This semaphore is suitable for all threads in the same thread block.
+     cuda::binary_semaphore<cuda::thread_scope_block> d;
+   }
+
+`See it on Godbolt <https://godbolt.org/z/eKfjYYz58>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/counting_semaphore.rst

@@ -0,0 +1,72 @@
+.. _libcudacxx-extended-api-synchronization-counting-semaphore:
+
+``cuda::counting_semaphore``
+============================
+
+Defined in header ``<cuda/semaphore>``:
+
+.. code:: cuda
+
+   template <cuda::thread_scope Scope,
+             cuda::std::ptrdiff_t LeastMaxValue = /* implementation-defined */>
+   class cuda::counting_semaphore;
+
+The class template ``cuda::counting_semaphore`` is an extended form of `cuda::std::counting_semaphore <https://en.cppreference.com/w/cpp/thread/counting_semaphore>`_
+that takes an additional :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` argument.
+``cuda::counting_semaphore`` has the same interface and semantics as
+`cuda::std::counting_semaphore <https://en.cppreference.com/w/cpp/thread/counting_semaphore>`_.
+
+Concurrency Restrictions
+------------------------
+
+An object of type ``cuda::counting_semaphore`` or ``cuda::std::counting_semaphore``, shall not be accessed concurrently
+by CPU and GPU threads unless:
+
+   - it is in unified memory and the `concurrentManagedAccess property <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaDeviceProp.html#structcudaDeviceProp_116f9619ccc85e93bc456b8c69c80e78b>`_
+     is 1, or
+   - it is in CPU memory and the `hostNativeAtomicSupported property <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaDeviceProp.html#structcudaDeviceProp_1ef82fd7d1d0413c7d6f33287e5b6306f>`_
+     is 1.
+
+Note, for objects of scopes other than ``cuda::thread_scope_system`` this is a data-race, and therefore also prohibited
+regardless of memory characteristics.
+
+Under CUDA Compute Capability 6 (Pascal) or prior, an object of type ``cuda::counting_semaphore`` or
+``cuda::std::counting_semaphore`` may not be used.
+
+Implementation-Defined Behavior
+-------------------------------
+
+For each :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` ``S`` and least maximum value
+``V``, ``cuda::counting_semaphore<S, V>::max()`` is as follows:
+
+.. list-table::
+   :widths: 50 50
+   :header-rows: 0
+
+   * - :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` ``S``
+     - ``cuda::binary_semaphore<S>::max()``
+   * - Any thread scope
+     - ``cuda::std::numeric_limits<cuda::std::ptrdiff_t>::max()``
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/semaphore>
+
+   __global__ void example_kernel() {
+     // This semaphore is suitable for all threads in the system.
+     cuda::counting_semaphore<cuda::thread_scope_system> a;
+
+     // This semaphore has the same type as the previous one (`a`).
+     cuda::std::counting_semaphore<> b;
+
+     // This semaphore is suitable for all threads on the current processor (e.g. GPU).
+     cuda::counting_semaphore<cuda::thread_scope_device> c;
+
+     // This semaphore is suitable for all threads in the same thread block.
+     cuda::counting_semaphore<cuda::thread_scope_block> d;
+   }
+
+`See it on Godbolt <https://godbolt.org/z/3YrjjTvG6>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/latch.rst

@@ -0,0 +1,70 @@
+.. _libcudacxx-extended-api-synchronization-latch:
+
+``cuda::latch``
+===============
+
+Defined in header ``<cuda/latch>``:
+
+.. code:: cpp
+
+   template <cuda::thread_scope Scope>
+   class cuda::latch;
+
+The class template ``cuda::latch`` is an extended form of `cuda::std::latch <https://en.cppreference.com/w/cpp/thread/latch>`_
+takes an additional :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` argument.
+It has the same interface and semantics as `cuda::std::latch <https://en.cppreference.com/w/cpp/thread/latch>`_.
+
+Concurrency Restrictions
+------------------------
+
+An object of type ``cuda::latch`` or `cuda::std::latch <https://en.cppreference.com/w/cpp/thread/latch>`_ shall not
+be accessed concurrently by CPU and GPU threads unless:
+
+   - it is in unified memory and the `concurrentManagedAccess property <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaDeviceProp.html#structcudaDeviceProp_116f9619ccc85e93bc456b8c69c80e78b>`_
+     is 1, or
+   - it is in CPU memory and the `hostNativeAtomicSupported property <https://docs.nvidia.com/cuda/cuda-runtime-api/structcudaDeviceProp.html#structcudaDeviceProp_1ef82fd7d1d0413c7d6f33287e5b6306f>`_
+     is 1.
+
+Note, for objects of scopes other than ``cuda::thread_scope_system`` this is a data-race, and therefore also prohibited
+regardless of memory characteristics.
+
+Under CUDA Compute Capability 6 (Pascal) or prior, an object of type ``cuda::latch`` or
+`cuda::std::latch <https://en.cppreference.com/w/cpp/thread/latch>`_ may not be used.
+
+Implementation-Defined Behavior
+-------------------------------
+
+For each :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` ``S``, the value of
+``cuda::latch<S>::max()`` is as follows:
+
+.. list-table::
+   :widths: 50 50
+   :header-rows: 0
+
+   * - :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` ``S``
+     - ``cuda::latch<S>::max()``
+   * - Any thread scope
+     - ``cuda::std::numeric_limits<cuda::std::ptrdiff_t>::max()``
+
+Example
+-------
+
+.. code:: cuda
+
+   #include <cuda/latch>
+
+   __global__ void example_kernel() {
+     // This latch is suitable for all threads in the system.
+     cuda::latch<cuda::thread_scope_system> a(10);
+
+     // This latch has the same type as the previous one (`a`).
+     cuda::std::latch b(10);
+
+     // This latch is suitable for all threads on the current processor (e.g. GPU).
+     cuda::latch<cuda::thread_scope_device> c(10);
+
+     // This latch is suitable for all threads in the same thread block.
+     cuda::latch<cuda::thread_scope_block> d(10);
+   }
+
+`See it on Godbolt <https://godbolt.org/z/8v4dcK7fa>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/pipeline.rst

@@ -0,0 +1,172 @@
+.. _libcudacxx-extended-api-synchronization-pipeline:
+
+``cuda::pipeline``
+==================
+
+.. toctree::
+   :hidden:
+   :maxdepth: 1
+
+   pipeline/role
+   pipeline/shared_state
+   pipeline/destructor
+   pipeline/make_pipeline
+   pipeline/quit
+   pipeline/consumer_release
+   pipeline/consumer_wait
+   pipeline/consumer_wait_prior
+   pipeline/pipeline_producer_commit
+   pipeline/producer_acquire
+   pipeline/producer_commit
+
+Defined in header ``<cuda/pipeline>``:
+
+.. code:: cuda
+
+   template <cuda::thread_scope Scope>
+   class cuda::pipeline {
+   public:
+     pipeline() = delete;
+
+     __host__ __device__ ~pipeline();
+
+     pipeline& operator=(pipeline const&) = delete;
+
+     __host__ __device__ void producer_acquire();
+
+     __host__ __device__ void producer_commit();
+
+     __host__ __device__ void consumer_wait();
+
+     template <typename Rep, typename Period>
+     __host__ __device__ bool consumer_wait_for(cuda::std::chrono::duration<Rep, Period> const& duration);
+
+     template <typename Clock, typename Duration>
+     __host__ __device__
+     bool consumer_wait_until(cuda::std::chrono::time_point<Clock, Duration> const& time_point);
+
+     __host__ __device__ void consumer_release();
+
+     __host__ __device__ bool quit();
+   };
+
+The class template ``cuda::pipeline`` provides a coordination mechanism which can sequence
+:ref:`asynchronous operations <libcudacxx-extended-api-asynchronous-operations>`, such as
+:ref:`cuda::memcpy_async <libcudacxx-extended-api-asynchronous-operations-memcpy-async>`, into stages.
+
+A thread interacts with a *pipeline stage* using the following pattern:
+
+  1. Acquire the pipeline stage.
+  2. Commit some operations to the stage.
+  3. Wait for the previously committed operations to complete.
+  4. Release the pipeline stage.
+
+For :ref:`cuda::thread_scope <libcudacxx-extended-api-memory-model-thread-scopes>` ``s`` other than
+``cuda::thread_scope_thread``, a
+:ref:`cuda::pipeline_shared_state <libcudacxx-extended-api-synchronization-pipeline-pipeline-shared-state>` is
+required to coordinate the participating threads.
+
+*Pipelines* can be either *unified* or *partitioned*. In a *unified pipeline*, all the participating threads are both
+producers and consumers. In a *partitioned pipeline*, each participating thread is either a producer or a consumer.
+
+.. rubric:: Template Parameters
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``Scope``
+     - The scope of threads participating in the *pipeline*.
+
+.. rubric:: Member Functions
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 1
+
+   * - Member function
+     - Description
+   * - (constructor) [deleted]
+     - ``cuda::pipeline`` is not constructible.
+   * - ``operator=`` [deleted]
+     - ``cuda::pipeline`` is not assignable.
+   * - :ref:`(destructor) <libcudacxx-extended-api-synchronization-pipeline-pipeline-destructor>`
+     -  Destroys the ``cuda::pipeline``.
+   * - :ref:`producer_acquire <libcudacxx-extended-api-synchronization-pipeline-pipeline-producer-acquire>`
+     - Blocks the current thread until the next *pipeline stage* is available.
+   * - :ref:`producer_commit <libcudacxx-extended-api-synchronization-pipeline-pipeline-producer-commit>`
+     - Commits operations previously issued by the current thread to the current *pipeline stage*.
+   * - :ref:`consumer_wait <libcudacxx-extended-api-synchronization-pipeline-pipeline-consumer-wait>`
+     - Blocks the current thread until all operations committed to the current *pipeline stage* complete.
+   * - :ref:`consumer_wait_for <libcudacxx-extended-api-synchronization-pipeline-pipeline-consumer-wait>`
+     - Blocks the current thread until all operations committed to the current *pipeline stage* complete or after the
+       specified timeout duration.
+   * - :ref:`consumer_wait_until <libcudacxx-extended-api-synchronization-pipeline-pipeline-consumer-wait>`
+     - Blocks the current thread until all operations committed to the current *pipeline stage* complete or until
+       specified time point has been reached.
+   * - :ref:`consumer_release <libcudacxx-extended-api-synchronization-pipeline-pipeline-consumer-release>`
+     - Release the current *pipeline stage*.
+   * - :ref:`quit <libcudacxx-extended-api-synchronization-pipeline-pipeline-quit>`
+     - Quits current thread's participation in the *pipeline*.
+
+.. note::
+
+   - A thread role cannot change during the lifetime of the pipeline object.
+
+.. rubric:: Example
+
+.. code:: cuda
+
+   #include <cuda/pipeline>
+   #include <cooperative_groups.h>
+
+   // Disables `pipeline_shared_state` initialization warning.
+   #pragma nv_diag_suppress static_var_with_dynamic_init
+
+   template <typename T>
+   __device__ void compute(T* ptr);
+
+   template <typename T>
+   __global__ void example_kernel(T* global0, T* global1, cuda::std::size_t subset_count) {
+     extern __shared__ T s[];
+     auto group = cooperative_groups::this_thread_block();
+     T* shared[2] = { s, s + 2 * group.size() };
+
+     // Create a pipeline.
+     constexpr auto scope = cuda::thread_scope_block;
+     constexpr auto stages_count = 2;
+     __shared__ cuda::pipeline_shared_state<scope, stages_count> shared_state;
+     auto pipeline = cuda::make_pipeline(group, &shared_state);
+
+     // Prime the pipeline.
+     pipeline.producer_acquire();
+     cuda::memcpy_async(group, shared[0],
+                        &global0[0], sizeof(T) * group.size(), pipeline);
+     cuda::memcpy_async(group, shared[0] + group.size(),
+                        &global1[0], sizeof(T) * group.size(), pipeline);
+     pipeline.producer_commit();
+
+     // Pipelined copy/compute.
+     for (cuda::std::size_t subset = 1; subset < subset_count; ++subset) {
+       pipeline.producer_acquire();
+       cuda::memcpy_async(group, shared[subset % 2],
+                          &global0[subset * group.size()],
+                          sizeof(T) * group.size(), pipeline);
+       cuda::memcpy_async(group, shared[subset % 2] + group.size(),
+                          &global1[subset * group.size()],
+                          sizeof(T) * group.size(), pipeline);
+       pipeline.producer_commit();
+       pipeline.consumer_wait();
+       compute(shared[(subset - 1) % 2]);
+       pipeline.consumer_release();
+     }
+
+     // Drain the pipeline.
+     pipeline.consumer_wait();
+     compute(shared[(subset_count - 1) % 2]);
+     pipeline.consumer_release();
+   }
+
+   template void __global__ example_kernel<int>(int*, int*, cuda::std::size_t);
+
+`See it on Godbolt <https://godbolt.org/z/zc41bWvja>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/pipeline/consumer_release.rst

@@ -0,0 +1,20 @@
+.. _libcudacxx-extended-api-synchronization-pipeline-pipeline-consumer-release:
+
+cuda::pipeline::consumer_release
+====================================
+
+Defined in header ``<cuda/pipeline>``:
+
+.. code:: cuda
+
+   template <cuda::thread_scope Scope>
+   __host__ __device__
+   void cuda::pipeline<Scope>::consumer_release();
+
+Releases the current *pipeline stage*.
+
+.. note::
+
+   - If the calling thread is a *producer thread*, the behavior is undefined.
+   - If the pipeline is in a :ref:`quitted state <libcudacxx-extended-api-synchronization-pipeline-pipeline-quit>`,
+     the behavior is undefined.

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/pipeline/consumer_wait.rst

@@ -0,0 +1,55 @@
+.. _libcudacxx-extended-api-synchronization-pipeline-pipeline-consumer-wait:
+
+cuda::pipeline::consumer_wait
+=================================
+
+Defined in header ``<cuda/pipeline>``:
+
+.. code:: cuda
+
+   // (1)
+   template <cuda::thread_scope Scope>
+   __host__ __device__
+   void cuda::pipeline<Scope>::consumer_wait();
+
+   // (2)
+   template <cuda::thread_scope Scope>
+   template <typename Rep, typename Period>
+   __host__ __device__
+   bool cuda::pipeline<Scope>::consumer_wait_for(
+     cuda::std::chrono::duration<Rep, Period> const& duration);
+
+   // (3)
+   template <cuda::thread_scope Scope>
+   template <typename Clock, typename Duration>
+   __host__ __device__
+   bool cuda::pipeline<Scope>::consumer_wait_until(
+     cuda::std::chrono::time_point<Clock, Duration> const& time_point);
+
+1. Blocks the current thread until all operations committed to the current *pipeline stage* complete.
+2. Blocks the current thread until all operations committed to the current *pipeline stage* complete or after the
+   specified timeout duration.
+3. Blocks the current thread until all operations committed to the current *pipeline stage* complete or until specified
+   time point has been reached.
+
+.. rubric:: Parameters
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``duration``
+     - An object of type ``cuda::std::chrono::duration`` representing the maximum time to spend waiting.
+   * - ``time_point``
+     - An object of type ``cuda::std::chrono::time_point`` representing the time when to stop waiting.
+
+
+.. rubric:: Return Value
+
+``false`` if the *wait* timed out, ``true`` otherwise.
+
+.. note::
+
+   - If the calling thread is a *producer thread*, the behavior is undefined.
+   - If the pipeline is in a :ref:`quitted state <libcudacxx-extended-api-synchronization-pipeline-pipeline-quit>`,
+     the behavior is undefined.

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/pipeline/consumer_wait_prior.rst

@@ -0,0 +1,66 @@
+.. _libcudacxx-extended-api-synchronization-pipeline-pipeline-consumer-wait-prior:
+
+cuda::pipeline_consumer_wait_prior
+======================================
+
+Defined in header ``<cuda/pipeline>``:
+
+.. code:: cuda
+
+   template <cuda::std::uint8_t Prior>
+   __host__ __device__
+   void cuda::pipeline_consumer_wait_prior(cuda::pipeline<thread_scope_thread>& pipe);
+
+Let *Stage* be the pipeline stage ``Prior`` stages before the current one (counting the current one).
+Blocks the current thread until all operations committed to *pipeline stages* up to *Stage* complete.
+All stages up to *Stage* (exclusive) are implicitly released.
+
+.. rubric:: Template Parameters
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``Prior``
+     - The index of the pipeline stage *Stage* (see above) counting up from the current one. The index of the current stage is ``0``.
+
+.. rubric:: Parameters
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``pipe``
+     - The thread-scoped ``cuda::pipeline`` object to wait on.
+
+.. note::
+
+   - If the pipeline is in a :ref:`quitted state <libcudacxx-extended-api-synchronization-pipeline-pipeline-quit>`,
+     the behavior is undefined.
+
+.. rubric:: Example
+
+.. code:: cuda
+
+   #include <cuda/pipeline>
+
+   __global__ void example_kernel(uint64_t* global, cuda::std::size_t element_count) {
+     extern __shared__ uint64_t shared[];
+
+     cuda::pipeline<cuda::thread_scope_thread> pipe = cuda::make_pipeline();
+     for (cuda::std::size_t i = 0; i < element_count; ++i) {
+       pipe.producer_acquire();
+       cuda::memcpy_async(shared + i, global + i, sizeof(*global), pipe);
+       pipe.producer_commit();
+     }
+
+     // Wait for operations committed in all stages but the last one.
+     cuda::pipeline_consumer_wait_prior<1>(pipe);
+     pipe.consumer_release();
+
+     // Wait for operations committed in all stages.
+     cuda::pipeline_consumer_wait_prior<0>(pipe);
+     pipe.consumer_release();
+   }
+
+`See it on Godbolt <https://godbolt.org/z/aT5hb84PY>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/pipeline/destructor.rst

@@ -0,0 +1,15 @@
+.. _libcudacxx-extended-api-synchronization-pipeline-pipeline-destructor:
+
+cuda::pipeline::~pipeline
+=============================
+
+Defined in header ``<cuda/pipeline>``:
+
+.. code:: cuda
+
+   template <cuda::thread_scope Scope>
+   __host__ __device__
+   cuda::pipeline<Scope>::~pipeline();
+
+Destructs the pipeline. Calls :ref:`cuda::pipeline::quit <libcudacxx-extended-api-synchronization-pipeline-pipeline-quit>`
+if it was not called by the current thread and destructs the pipeline.

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/pipeline/make_pipeline.rst

@@ -0,0 +1,125 @@
+.. _libcudacxx-extended-api-synchronization-pipeline-pipeline-make-pipeline:
+
+cuda::make_pipeline
+=======================
+
+Defined in header ``<cuda/pipeline>``:
+
+.. code:: cuda
+
+   // (1)
+   __host__ __device__
+   cuda::pipeline<cuda::thread_scope_thread> cuda::make_pipeline();
+
+   // (2)
+   template <typename Group,
+             cuda::thread_scope Scope,
+             cuda::std::uint8_t StagesCount>
+   __host__ __device__
+   cuda::pipeline<Scope>
+   cuda::make_pipeline(Group const& group,
+                       cuda::pipeline_shared_state<Scope, StagesCount>* shared_state);
+
+   // (3)
+   template <typename Group,
+             cuda::thread_scope Scope,
+             cuda::std::uint8_t StagesCount>
+   __host__ __device__
+   cuda::pipeline<Scope>
+   cuda::make_pipeline(Group const& group,
+                       cuda::pipeline_shared_state<Scope, StagesCount>* shared_state,
+                       cuda::std::size_t producer_count);
+
+   // (4)
+   template <typename Group,
+             cuda::thread_scope Scope,
+             cuda::std::uint8_t StagesCount>
+   __host__ __device__
+   cuda::pipeline<Scope>
+   cuda::make_pipeline(Group const& group,
+                       cuda::pipeline_shared_state<Scope, StagesCount>* shared_state,
+                       cuda::pipeline_role role);
+
+1. Creates a *unified pipeline* such that the calling thread is the only participating thread and performs both
+   producer and consumer actions.
+2. Creates a *unified pipeline* such that all the threads in ``group`` are performing both producer and consumer actions.
+3. Creates a *partitioned pipeline* such that ``producer_threads`` number of threads in ``group`` are performing
+   producer actions while the others are performing consumer actions.
+4. Creates a *partitioned pipeline* where each thread's role is explicitly specified.
+
+.. rubric:: Template Parameters
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``Group``
+     - A type satisfying the :ref:`ThreadGroup <libcudacxx-extended-api-thread-groups>` concept.
+
+.. rubric:: Parameters
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``group``
+     - The group of threads.
+   * - ``shared_state``
+     - A pointer to an object of type :ref:`cuda::pipeline_shared_state\<Scope\> <libcudacxx-extended-api-synchronization-pipeline-pipeline-shared-state>`
+       with ``Scope`` including all the threads in ``group``.
+   * - ``producer_count``
+     - The number of *producer threads* in the pipeline.
+   * - ``role``
+     - The role of the current thread in the pipeline.
+
+.. rubric:: Return Value
+
+A ``cuda::pipeline`` object.
+
+.. note::
+
+  - All threads in ``group`` acquire collective ownership of the ``shared_state`` storage.
+  - ``make_pipeline`` must be invoked by every threads in ``group`` such that ``group::sync`` may be invoked.
+  - ``shared_state`` and ``producer_count`` must be the same across all threads in ``group``, else the behavior is undefined.
+  - ``producer_count`` must be strictly inferior to ``group::size``, otherwise the behavior is undefined.
+
+.. _libcudacxx-extended-api-synchronization-pipeline-pipeline-make-pipeline-example:
+
+.. rubric:: Example
+
+.. code:: cuda
+
+   #include <cuda/pipeline>
+   #include <cooperative_groups.h>
+
+   // Disables `pipeline_shared_state` initialization warning.
+   #pragma nv_diag_suppress static_var_with_dynamic_init
+
+   __global__ void example_kernel() {
+     __shared__ cuda::pipeline_shared_state<cuda::thread_scope_block, 2> pss0;
+     __shared__ cuda::pipeline_shared_state<cuda::thread_scope_block, 2> pss1;
+     __shared__ cuda::pipeline_shared_state<cuda::thread_scope_block, 2> pss2;
+
+     auto group = cooperative_groups::this_thread_block();
+
+     // Create a single thread scoped pipeline.
+     cuda::pipeline<cuda::thread_scope_thread> p0 = cuda::make_pipeline();
+
+     // Create a unified block-scoped pipeline.
+     cuda::pipeline<cuda::thread_scope_block> p1 = cuda::make_pipeline(group, &pss0);
+
+     // Create a partitioned block-scoped pipeline where half the threads are producers.
+     cuda::std::size_t producer_count = group.size() / 2;
+     cuda::pipeline<cuda::thread_scope_block> p2
+       = cuda::make_pipeline(group, &pss1, producer_count);
+
+     // Create a partitioned block-scoped pipeline where all threads with an even
+     // `thread_rank` are producers.
+     auto thread_role = (group.thread_rank() % 2)
+                        ? cuda::pipeline_role::producer
+                       : cuda::pipeline_role::consumer;
+     cuda::pipeline<cuda::thread_scope_block> p3
+       = cuda::make_pipeline(group, &pss2, thread_role);
+   }
+
+`See it on Godbolt <https://godbolt.org/z/aPcGEr64j>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/pipeline/pipeline_producer_commit.rst

@@ -0,0 +1,60 @@
+.. _libcudacxx-extended-api-synchronization-pipeline-pipeline-pipeline-producer-commit:
+
+cuda::pipeline_producer_commit
+==================================
+
+Defined in header ``<cuda/pipeline>``:
+
+.. code:: cuda
+
+   template <cuda::thread_scope Scope>
+   __host__ __device__
+   void cuda::pipeline_producer_commit(cuda::pipeline<cuda::thread_scope_thread>& pipe,
+                                       cuda::barrier<Scope>& bar);
+
+Binds operations previously issued by the current thread to the named ``cuda::barrier`` such that a
+``cuda::barrier::arrive`` is performed on completion. The bind operation implicitly increments the barrier's
+current phase to account for the subsequent ``cuda::barrier::arrive``, resulting in a net change of 0.
+
+.. rubric:: Parameters
+
+.. list-table::
+   :widths: 25 75
+   :header-rows: 0
+
+   * - ``pipe``
+     - The thread-scoped ``cuda::pipeline`` object to wait on.
+   * - ``bar``
+     - The ``cuda::barrier`` to arrive on.
+
+.. note::
+
+   - If the pipeline is in a :ref:`quitted state <libcudacxx-extended-api-synchronization-pipeline-pipeline-quit>`,
+     the behavior is undefined.
+
+.. rubric:: Example
+
+.. code:: cuda
+
+   #include <cuda/pipeline>
+
+   // Disables `barrier` initialization warning.
+   #pragma nv_diag_suppress static_var_with_dynamic_init
+
+   __global__ void
+   example_kernel(cuda::std::uint64_t* global, cuda::std::size_t element_count) {
+     extern __shared__ cuda::std::uint64_t shared[];
+     __shared__ cuda::barrier<cuda::thread_scope_block> barrier;
+
+     init(&barrier, 1);
+     cuda::pipeline<cuda::thread_scope_thread> pipe = cuda::make_pipeline();
+
+     pipe.producer_acquire();
+     for (cuda::std::size_t i = 0; i < element_count; ++i)
+       cuda::memcpy_async(shared + i, global + i, sizeof(*global), pipe);
+     pipeline_producer_commit(pipe, barrier);
+     barrier.arrive_and_wait();
+     pipe.consumer_release();
+   }
+
+`See it on Godbolt <https://godbolt.org/z/sGzKe9obf>`_

cccl_upstream/docs/libcudacxx/extended_api/synchronization_primitives/pipeline/producer_acquire.rst

@@ -0,0 +1,20 @@
+.. _libcudacxx-extended-api-synchronization-pipeline-pipeline-producer-acquire:
+
+cuda::pipeline::producer_acquire
+====================================
+
+Defined in header ``<cuda/pipeline>``:
+
+.. code:: cuda
+
+   template <cuda::thread_scope Scope>
+   __host__ __device__
+   void cuda::pipeline<Scope>::producer_acquire();
+
+Blocks the current thread until the next *pipeline stage* is available.
+
+.. note::
+
+   - If the calling thread is a *consumer thread*, the behavior is undefined.
+   - If the pipeline is in a :ref:`quitted state <libcudacxx-extended-api-synchronization-pipeline-pipeline-quit>`,
+     the behavior is undefined.