[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/libcudacxx/include/cuda/__annotated_ptr/access_property.h

@@ -0,0 +1,170 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___ANNOTATED_PTR_ACCESS_PROPERTY_H
+#define _CUDA___ANNOTATED_PTR_ACCESS_PROPERTY_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__annotated_ptr/access_property_encoding.h>
+#include <cuda/std/cstddef>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename>
+class __annotated_ptr_base; // forward declaration
+
+class access_property
+{
+private:
+  uint64_t __descriptor = __l2_interleave_normal;
+
+  friend class __annotated_ptr_base<access_property>;
+
+  // needed by __annotated_ptr_base
+  _CCCL_HOST_DEVICE_API constexpr access_property(uint64_t __descriptor1) noexcept
+      : __descriptor{__descriptor1}
+  {}
+
+public:
+  struct shared
+  {};
+  struct global
+  {};
+  struct persisting
+  {
+#if _CCCL_HAS_CTK()
+    [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr operator ::cudaAccessProperty() const noexcept
+    {
+      return ::cudaAccessProperty::cudaAccessPropertyPersisting;
+    }
+#endif // _CCCL_HAS_CTK()
+  };
+  struct streaming
+  {
+#if _CCCL_HAS_CTK()
+    [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr operator ::cudaAccessProperty() const noexcept
+    {
+      return ::cudaAccessProperty::cudaAccessPropertyStreaming;
+    }
+#endif // _CCCL_HAS_CTK()
+  };
+  struct normal
+  {
+#if _CCCL_HAS_CTK()
+    [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr operator ::cudaAccessProperty() const noexcept
+    {
+      return ::cudaAccessProperty::cudaAccessPropertyNormal;
+    }
+#endif // _CCCL_HAS_CTK()
+  };
+
+  _CCCL_HIDE_FROM_ABI access_property() noexcept = default;
+
+  _CCCL_HOST_DEVICE_API constexpr access_property(normal, float __fraction) noexcept
+      : __descriptor{
+          ::cuda::__l2_interleave(__l2_evict_t::_L2_Evict_Normal_Demote, __l2_evict_t::_L2_Evict_Unchanged, __fraction)}
+  {}
+  _CCCL_HOST_DEVICE_API constexpr access_property(streaming, float __fraction) noexcept
+      : __descriptor{
+          ::cuda::__l2_interleave(__l2_evict_t::_L2_Evict_First, __l2_evict_t::_L2_Evict_Unchanged, __fraction)}
+  {}
+  _CCCL_HOST_DEVICE_API constexpr access_property(persisting, float __fraction) noexcept
+      : __descriptor{
+          ::cuda::__l2_interleave(__l2_evict_t::_L2_Evict_Last, __l2_evict_t::_L2_Evict_Unchanged, __fraction)}
+  {}
+  _CCCL_HOST_DEVICE_API constexpr access_property(normal, float __fraction, streaming) noexcept
+      : __descriptor{
+          ::cuda::__l2_interleave(__l2_evict_t::_L2_Evict_Normal_Demote, __l2_evict_t::_L2_Evict_First, __fraction)}
+  {}
+  _CCCL_HOST_DEVICE_API constexpr access_property(persisting, float __fraction, streaming) noexcept
+      : __descriptor{::cuda::__l2_interleave(__l2_evict_t::_L2_Evict_Last, __l2_evict_t::_L2_Evict_First, __fraction)}
+  {}
+
+  _CCCL_HOST_DEVICE_API constexpr access_property(global) noexcept {}
+
+  _CCCL_HOST_DEVICE_API constexpr access_property(normal) noexcept
+      : access_property{normal{}, 1.0f}
+  {}
+  _CCCL_HOST_DEVICE_API constexpr access_property(streaming) noexcept
+      : access_property{streaming{}, 1.0f}
+  {}
+  _CCCL_HOST_DEVICE_API constexpr access_property(persisting) noexcept
+      : access_property{persisting{}, 1.0f}
+  {}
+
+  _CCCL_HOST_DEVICE_API inline access_property(
+    void* __ptr, size_t __primary_bytes, size_t __total_bytes, normal) noexcept
+      : __descriptor{::cuda::__block_encoding(
+          __l2_evict_t::_L2_Evict_Normal_Demote,
+          __l2_evict_t::_L2_Evict_Unchanged,
+          __ptr,
+          __primary_bytes,
+          __total_bytes)}
+  {}
+
+  _CCCL_HOST_DEVICE_API inline access_property(
+    void* __ptr, size_t __primary_bytes, size_t __total_bytes, streaming) noexcept
+      : __descriptor{::cuda::__block_encoding(
+          __l2_evict_t::_L2_Evict_First, __l2_evict_t::_L2_Evict_Unchanged, __ptr, __primary_bytes, __total_bytes)}
+  {}
+
+  _CCCL_HOST_DEVICE_API inline access_property(
+    void* __ptr, size_t __primary_bytes, size_t __total_bytes, persisting) noexcept
+      : __descriptor{::cuda::__block_encoding(
+          __l2_evict_t::_L2_Evict_Last, __l2_evict_t::_L2_Evict_Unchanged, __ptr, __primary_bytes, __total_bytes)}
+  {}
+
+  _CCCL_HOST_DEVICE_API inline access_property(
+    void* __ptr, size_t __primary_bytes, size_t __total_bytes, global, streaming) noexcept
+      : __descriptor{::cuda::__block_encoding(
+          __l2_evict_t::_L2_Evict_Unchanged, __l2_evict_t::_L2_Evict_First, __ptr, __primary_bytes, __total_bytes)}
+  {}
+
+  _CCCL_HOST_DEVICE_API inline access_property(
+    void* __ptr, size_t __primary_bytes, size_t __total_bytes, normal, streaming) noexcept
+      : __descriptor{::cuda::__block_encoding(
+          __l2_evict_t::_L2_Evict_Normal_Demote, __l2_evict_t::_L2_Evict_First, __ptr, __primary_bytes, __total_bytes)}
+  {}
+
+  _CCCL_HOST_DEVICE_API inline access_property(
+    void* __ptr, size_t __primary_bytes, size_t __total_bytes, streaming, streaming) noexcept
+      : __descriptor{::cuda::__block_encoding(
+          __l2_evict_t::_L2_Evict_First, __l2_evict_t::_L2_Evict_First, __ptr, __primary_bytes, __total_bytes)}
+  {}
+
+  _CCCL_HOST_DEVICE_API inline access_property(
+    void* __ptr, size_t __primary_bytes, size_t __total_bytes, persisting, streaming) noexcept
+      : __descriptor{::cuda::__block_encoding(
+          __l2_evict_t::_L2_Evict_Last, __l2_evict_t::_L2_Evict_First, __ptr, __primary_bytes, __total_bytes)}
+  {}
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr explicit operator uint64_t() const noexcept
+  {
+    return __descriptor;
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___ANNOTATED_PTR_ACCESS_PROPERTY_H

cccl_upstream/libcudacxx/include/cuda/__annotated_ptr/access_property_encoding.h

@@ -0,0 +1,171 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___ANNOTATED_PTR_ACCESS_PROPERTY_ENCODING_H
+#define _CUDA___ANNOTATED_PTR_ACCESS_PROPERTY_ENCODING_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__annotated_ptr/createpolicy.h>
+#include <cuda/__cmath/ilog.h>
+#include <cuda/std/__algorithm/clamp.h>
+#include <cuda/std/__algorithm/max.h>
+#include <cuda/std/__bit/bit_cast.h>
+#include <cuda/std/__numeric/saturating_sub.h>
+#include <cuda/std/__utility/to_underlying.h>
+#include <cuda/std/cstddef>
+#include <cuda/std/cstdint>
+#include <cuda/std/limits>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+enum class __l2_descriptor_mode_t : uint32_t
+{
+  _Desc_Implicit    = 0,
+  _Desc_Interleaved = 2,
+  _Desc_Block_Type  = 3
+};
+
+/***********************************************************************************************************************
+ * Range Block Descriptor
+ **********************************************************************************************************************/
+
+// MemoryDescriptor:blockDesc_t  reference
+//
+// struct __block_desc_t // 64 bits
+// {
+//   uint64_t __reserved1       : 37;
+//   uint32_t __block_count     : 7;
+//   uint32_t __block_start     : 7;
+//   uint32_t __reserved2       : 1;
+//   uint32_t __block_size_enum : 4; // 56 bits
+//
+//   uint32_t __l2_cop_off             : 1;
+//   uint32_t __l2_cop_on              : 2;
+//   uint32_t __l2_descriptor_mode     : 2;
+//   uint32_t __l1_inv_dont_allocate   : 1;
+//   uint32_t __l2_sector_promote_256B : 1;
+//   uint32_t __reserved3              : 1;
+// };
+
+#if !_CCCL_CUDA_COMPILER(NVRTC)
+
+[[nodiscard]] _CCCL_HOST_API inline uint64_t __block_encoding_host(
+  __l2_evict_t __primary, __l2_evict_t __secondary, const void* __ptr, uint32_t __primary_bytes, uint32_t __total_bytes)
+{
+  _CCCL_ASSERT(__primary_bytes > 0, "primary_size must be greater than 0");
+  _CCCL_ASSERT(__primary_bytes <= __total_bytes, "primary_size must be less than or equal to total_size");
+  _CCCL_ASSERT(__secondary == __l2_evict_t::_L2_Evict_First || __secondary == __l2_evict_t::_L2_Evict_Unchanged,
+               "secondary policy must be evict_first or evict_unchanged");
+  auto __raw_ptr         = ::cuda::std::bit_cast<uintptr_t>(__ptr);
+  auto __log2_total_size = ::cuda::ceil_ilog2(__total_bytes);
+  auto __block_size_enum = ::cuda::std::saturating_sub<uint32_t>(__log2_total_size, 19); // min block size = 4K
+  auto __log2_block_size = 12u + __block_size_enum;
+  auto __block_size      = 1u << __log2_block_size;
+  auto __block_start     = static_cast<uint32_t>(__raw_ptr >> __log2_block_size); // ptr / block_size
+  // vvvv block_end = ceil_div(ptr + primary_size, block_size)
+  auto __block_end = static_cast<uint32_t>((__raw_ptr + __primary_bytes + __block_size - 1) >> __log2_block_size);
+  _CCCL_ASSERT(__block_end >= __block_start, "block_end < block_start");
+  // NOTE: there is a bug in PTX createpolicy when __block_size_enum == 13. The *incorrect* behavior matches the
+  //       following code:
+  // auto __block_count        = (__block_size_enum == 13)
+  //                            ? ((__block_end - __block_start <= 127u) ? (__block_end - __block_start) : 1)
+  //                            : ::cuda::std::clamp(__block_end - __block_start, 1u, 127u);
+  auto __block_count        = ::cuda::std::clamp(__block_end - __block_start, 1u, 127u);
+  auto __l2_cop_off         = ::cuda::std::to_underlying(__secondary);
+  auto __l2_cop_on          = ::cuda::std::to_underlying(__primary);
+  auto __l2_descriptor_mode = ::cuda::std::to_underlying(__l2_descriptor_mode_t::_Desc_Block_Type);
+  return static_cast<uint64_t>(__block_count) << 37 //
+       | static_cast<uint64_t>(__block_start) << 44 //
+       | static_cast<uint64_t>(__block_size_enum) << 52 //
+       | static_cast<uint64_t>(__l2_cop_off) << 56 //
+       | static_cast<uint64_t>(__l2_cop_on) << 57 //
+       | static_cast<uint64_t>(__l2_descriptor_mode) << 59;
+}
+
+#endif // !_CCCL_CUDA_COMPILER(NVRTC)
+
+[[nodiscard]] _CCCL_HOST_DEVICE_API inline uint64_t __block_encoding(
+  __l2_evict_t __primary, __l2_evict_t __secondary, const void* __ptr, size_t __primary_bytes, size_t __total_bytes)
+{
+  _CCCL_ASSERT(__primary_bytes <= size_t{0xFFFFFFFF}, "primary size must be less than 4GB");
+  _CCCL_ASSERT(__total_bytes <= size_t{0xFFFFFFFF}, "total size must be less than 4GB");
+  auto __primary_bytes1 = static_cast<uint32_t>(__primary_bytes);
+  auto __total_bytes1   = static_cast<uint32_t>(__total_bytes);
+  NV_IF_ELSE_TARGET(
+    NV_IS_HOST,
+    (return ::cuda::__block_encoding_host(__primary, __secondary, __ptr, __primary_bytes1, __total_bytes1);),
+    (return ::cuda::__createpolicy_range(__primary, __secondary, __ptr, __primary_bytes1, __total_bytes1);))
+}
+
+/***********************************************************************************************************************
+ * Interleaved Descriptor
+ **********************************************************************************************************************/
+
+// MemoryDescriptor:interleaveDesc_t reference
+//
+// struct __interleaved_desc_t // 64 bits
+// {
+//   uint64_t            : 52;
+//   uint32_t __fraction : 4; // 56 bits
+//
+//   uint32_t __l2_cop_off             : 1;
+//   uint32_t __l2_cop_on              : 2;
+//   uint32_t __l2_descriptor_mode     : 2;
+//   uint32_t __l1_inv_dont_allocate   : 1;
+//   uint32_t __l2_sector_promote_256B : 1;
+//   uint32_t                          : 1;
+// };
+
+[[nodiscard]] _CCCL_HOST_DEVICE_API constexpr uint64_t
+__l2_interleave(__l2_evict_t __primary, __l2_evict_t __secondary, float __fraction)
+{
+  _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+  {
+    NV_IF_ELSE_TARGET(
+      NV_PROVIDES_SM_80, (return ::cuda::__createpolicy_fraction(__primary, __secondary, __fraction);), (return 0;))
+  }
+  _CCCL_ASSERT(__fraction > 0.0f && __fraction <= 1.0f, "fraction must be between 0.0f and 1.0f");
+  _CCCL_ASSERT(__secondary == __l2_evict_t::_L2_Evict_First || __secondary == __l2_evict_t::_L2_Evict_Unchanged,
+               "secondary policy must be evict_first or evict_unchanged");
+  constexpr auto __epsilon  = ::cuda::std::numeric_limits<float>::epsilon();
+  auto __num                = static_cast<uint32_t>((__fraction - __epsilon) * 16.0f); // fraction = num / 16
+  auto __l2_cop_off         = ::cuda::std::to_underlying(__secondary);
+  auto __l2_cop_on          = ::cuda::std::to_underlying(__primary);
+  auto __l2_descriptor_mode = ::cuda::std::to_underlying(__l2_descriptor_mode_t::_Desc_Interleaved);
+  return static_cast<uint64_t>(__num) << 52 //
+       | static_cast<uint64_t>(__l2_cop_off) << 56 //
+       | static_cast<uint64_t>(__l2_cop_on) << 57 //
+       | static_cast<uint64_t>(__l2_descriptor_mode) << 59;
+}
+
+inline constexpr auto __l2_interleave_normal = uint64_t{0x10F0000000000000};
+
+inline constexpr auto __l2_interleave_streaming = uint64_t{0x12F0000000000000};
+
+inline constexpr auto __l2_interleave_persisting = uint64_t{0x14F0000000000000};
+
+inline constexpr auto __l2_interleave_normal_demote = uint64_t{0x16F0000000000000};
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___ANNOTATED_PTR_ACCESS_PROPERTY_ENCODING_H

cccl_upstream/libcudacxx/include/cuda/__annotated_ptr/annotated_ptr.h

@@ -0,0 +1,216 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___ANNOTATED_PTR_ANNOTATED_PTR_H
+#define _CUDA___ANNOTATED_PTR_ANNOTATED_PTR_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__annotated_ptr/access_property.h>
+#include <cuda/__annotated_ptr/annotated_ptr_base.h>
+#include <cuda/__memcpy_async/memcpy_async.h>
+#include <cuda/__memory/address_space.h>
+#include <cuda/std/cstddef>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _Tp, typename _Property>
+class annotated_ptr : private ::cuda::__annotated_ptr_base<_Property>
+{
+public:
+  using value_type      = _Tp;
+  using size_type       = size_t;
+  using reference       = value_type&;
+  using pointer         = value_type*;
+  using const_pointer   = const value_type*;
+  using difference_type = ptrdiff_t;
+
+private:
+  static_assert(__is_access_property_v<_Property>);
+
+  static constexpr bool __is_smem = ::cuda::std::is_same_v<_Property, access_property::shared>;
+
+  // Converting from a 64-bit to 32-bit shared pointer and maybe back just for storage might or might not be profitable.
+  pointer __repr = nullptr;
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API inline pointer __get(difference_type __n = 0) const noexcept
+  {
+    NV_IF_TARGET(NV_IS_DEVICE,
+                 (auto __repr1 = const_cast<void*>(static_cast<const volatile void*>(__repr + __n));
+                  return static_cast<pointer>(this->__apply_prop(__repr1));))
+    return __repr + __n;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API inline pointer __offset(difference_type __n) const noexcept
+  {
+    return __get(__n);
+  }
+
+public:
+  _CCCL_HIDE_FROM_ABI annotated_ptr() noexcept = default;
+
+  _CCCL_HOST_DEVICE_API explicit constexpr annotated_ptr(pointer __p) noexcept
+      : __repr{__p}
+  {
+    NV_IF_TARGET(NV_IS_HOST, (_CCCL_ASSERT(!__is_smem, "shared memory pointer is not supported on the host");))
+    if constexpr (__is_smem)
+    {
+      _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+      {
+        NV_IF_TARGET(NV_IS_DEVICE,
+                     (_CCCL_ASSERT(::cuda::device::is_address_from(__p, ::cuda::device::address_space::shared),
+                                   "__p must be shared");))
+      }
+    }
+    else
+    {
+      _CCCL_ASSERT(__p != nullptr, "__p must not be null");
+      _CCCL_IF_NOT_CONSTEVAL_DEFAULT
+      {
+        NV_IF_TARGET(NV_IS_DEVICE,
+                     (_CCCL_ASSERT(::cuda::device::is_address_from(__p, ::cuda::device::address_space::global),
+                                   "__p must be global");))
+      }
+    }
+  }
+
+  template <typename _RuntimeProperty>
+  _CCCL_HOST_DEVICE_API inline annotated_ptr(pointer __p, _RuntimeProperty __prop) noexcept
+      : ::cuda::__annotated_ptr_base<_Property>{access_property{__prop}}
+      , __repr{__p}
+  {
+    static_assert(::cuda::std::is_same_v<_Property, access_property>,
+                  "This method requires annotated_ptr<T, cuda::access_property>");
+    static_assert(__is_global_access_property_v<_RuntimeProperty>,
+                  "This method requires RuntimeProperty=global|normal|streaming|persisting|access_property");
+    _CCCL_ASSERT(__p != nullptr, "__p must not be null");
+    NV_IF_TARGET(NV_IS_DEVICE,
+                 (_CCCL_ASSERT(::cuda::device::is_address_from(__p, ::cuda::device::address_space::global),
+                               "__p must be global");))
+  }
+
+  // cannot be constexpr because of get()
+  template <typename _OtherType, class _OtherProperty>
+  _CCCL_HOST_DEVICE_API inline annotated_ptr(const annotated_ptr<_OtherType, _OtherProperty>& __other) noexcept
+      : ::cuda::__annotated_ptr_base<_Property>{__other.__property()}
+      , __repr{__other.get()}
+  {
+    using namespace ::cuda::std;
+    static_assert(is_assignable_v<pointer&, _OtherType*>, "pointer must be assignable from other pointer");
+    static_assert(is_same_v<_Property, _OtherProperty>
+                    || (is_same_v<_Property, access_property> && !is_same_v<_OtherProperty, access_property::shared>),
+                  "Both properties must have same address space, or current property is access_property and "
+                  "OtherProperty is not shared");
+  }
+
+  // cannot be constexpr because is_constant_evaluated is not supported by clang-14, gcc-8.
+  // when the method is called in these platforms, it needs to be called at run-time.
+  [[nodiscard]] _CCCL_HOST_DEVICE_API inline pointer operator->() const noexcept
+  {
+    return __get();
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API inline reference operator*() const noexcept
+  {
+    _CCCL_ASSERT(__get() != nullptr, "dereference of null annotated_ptr");
+    return *__get();
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API inline reference operator[](difference_type __n) const noexcept
+  {
+    _CCCL_ASSERT(__offset(__n) != nullptr, "dereference of null annotated_ptr");
+    return *__offset(__n);
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr difference_type operator-(annotated_ptr __other) const noexcept
+  {
+    _CCCL_ASSERT(__repr >= __other.__repr, "underflow");
+    return __repr - __other.__repr;
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr explicit operator bool() const noexcept
+  {
+    return (__repr != nullptr);
+  }
+
+  // cannot be constexpr because of operator->()
+  [[nodiscard]] _CCCL_HOST_DEVICE_API inline pointer get() const noexcept
+  {
+    return (__is_smem || __repr == nullptr)
+           ? __repr
+           : annotated_ptr<value_type, access_property::global>{__repr}.operator->();
+  }
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _Property __property() const noexcept
+  {
+    return this->__get_property();
+  }
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+// memcpy_async
+
+template <typename _Dst, typename _Src, typename _SrcProperty, typename _Shape, typename _Sync>
+_CCCL_HOST_DEVICE_API inline void
+memcpy_async(_Dst* __dst, annotated_ptr<_Src, _SrcProperty> __src, _Shape __shape, _Sync& __sync) noexcept
+{
+  ::cuda::memcpy_async(__dst, __src.operator->(), __shape, __sync);
+}
+
+template <typename _Dst, typename _DstProperty, typename _Src, typename _SrcProperty, typename _Shape, typename _Sync>
+_CCCL_HOST_DEVICE_API inline void memcpy_async(
+  annotated_ptr<_Dst, _DstProperty> __dst,
+  annotated_ptr<_Src, _SrcProperty> __src,
+  _Shape __shape,
+  _Sync& __sync) noexcept
+{
+  ::cuda::memcpy_async(__dst.operator->(), __src.operator->(), __shape, __sync);
+}
+
+template <typename _Group, typename _Dst, typename _Src, typename _SrcProperty, typename _Shape, typename _Sync>
+_CCCL_HOST_DEVICE_API inline void memcpy_async(
+  const _Group& __group, _Dst* __dst, annotated_ptr<_Src, _SrcProperty> __src, _Shape __shape, _Sync& __sync) noexcept
+{
+  ::cuda::memcpy_async(__group, __dst, __src.operator->(), __shape, __sync);
+}
+
+template <typename _Group,
+          typename _Dst,
+          typename _DstProperty,
+          typename _Src,
+          typename _SrcProperty,
+          typename _Shape,
+          typename _Sync>
+_CCCL_HOST_DEVICE_API inline void memcpy_async(
+  const _Group& __group,
+  annotated_ptr<_Dst, _DstProperty> __dst,
+  annotated_ptr<_Src, _SrcProperty> __src,
+  _Shape __shape,
+  _Sync& __sync) noexcept
+{
+  ::cuda::memcpy_async(__group, __dst.operator->(), __src.operator->(), __shape, __sync);
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___ANNOTATED_PTR_ANNOTATED_PTR_H

cccl_upstream/libcudacxx/include/cuda/__annotated_ptr/annotated_ptr_base.h

@@ -0,0 +1,100 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___ANNOTATED_PTR_ANNOTATED_PTR_BASE_H
+#define _CUDA___ANNOTATED_PTR_ANNOTATED_PTR_BASE_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__annotated_ptr/access_property.h>
+#include <cuda/__annotated_ptr/associate_access_property.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _AccessProperty>
+class __annotated_ptr_base
+{
+protected:
+  _CCCL_HOST_DEVICE_API static constexpr uint64_t __default_property() noexcept
+  {
+    return ::cuda::std::is_same_v<_AccessProperty, access_property::global>     ? __l2_interleave_normal
+         : ::cuda::std::is_same_v<_AccessProperty, access_property::normal>     ? __l2_interleave_normal_demote
+         : ::cuda::std::is_same_v<_AccessProperty, access_property::persisting> ? __l2_interleave_persisting
+         : ::cuda::std::is_same_v<_AccessProperty, access_property::streaming>
+           ? __l2_interleave_streaming
+           : 0; // access_property::shared;
+  }
+
+  static constexpr uint64_t __prop = __default_property();
+
+  _CCCL_HIDE_FROM_ABI __annotated_ptr_base() noexcept = default;
+
+  _CCCL_HOST_DEVICE_API constexpr __annotated_ptr_base(_AccessProperty) noexcept {}
+
+#if _CCCL_CUDA_COMPILATION()
+
+  [[nodiscard]] _CCCL_DEVICE_API void* __apply_prop(void* __p) const
+  {
+    return ::cuda::__associate(__p, _AccessProperty{});
+  }
+
+#endif // _CCCL_CUDA_COMPILATION()
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr _AccessProperty __get_property() const noexcept
+  {
+    return _AccessProperty{};
+  }
+};
+
+//----------------------------------------------------------------------------------------------------------------------
+// Specialization for dynamic access property
+
+template <>
+class __annotated_ptr_base<access_property>
+{
+protected:
+  uint64_t __prop = static_cast<uint64_t>(access_property{});
+
+  _CCCL_HOST_DEVICE_API constexpr __annotated_ptr_base(access_property __property) noexcept
+      : __prop{static_cast<uint64_t>(__property)}
+  {}
+
+  _CCCL_HIDE_FROM_ABI __annotated_ptr_base() noexcept = default;
+
+#if _CCCL_CUDA_COMPILATION()
+  [[nodiscard]] _CCCL_DEVICE_API void* __apply_prop(void* __p) const
+  {
+    return ::cuda::__associate_raw_descriptor(__p, __prop);
+  }
+#endif // _CCCL_CUDA_COMPILATION()
+
+  [[nodiscard]] _CCCL_HOST_DEVICE_API constexpr access_property __get_property() const noexcept
+  {
+    return access_property{__prop};
+  }
+};
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___ANNOTATED_PTR_ANNOTATED_PTR_BASE_H

cccl_upstream/libcudacxx/include/cuda/__annotated_ptr/apply_access_property.h

@@ -0,0 +1,83 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___ANNOTATED_PTR_APPLY_ACCESS_PROPERTY_H
+#define _CUDA___ANNOTATED_PTR_APPLY_ACCESS_PROPERTY_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__annotated_ptr/access_property.h>
+#include <cuda/__memory/address_space.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+template <typename _Shape>
+_CCCL_HOST_DEVICE_API inline void apply_access_property(
+  [[maybe_unused]] const volatile void* __ptr,
+  [[maybe_unused]] _Shape __shape,
+  [[maybe_unused]] access_property::persisting __prop) noexcept
+{
+  // clang-format off
+  NV_IF_TARGET(
+    NV_PROVIDES_SM_80,
+    (_CCCL_ASSERT(__ptr != nullptr, "null pointer");
+     if (!::cuda::device::is_address_from(__ptr, ::cuda::device::address_space::global))
+     {
+       return;
+     }
+     constexpr size_t __line_size = 128;
+     auto __p                     = reinterpret_cast<uint8_t*>(const_cast<void*>(__ptr));
+     auto __nbytes                = static_cast<size_t>(__shape);
+     // Apply to all 128 bytes aligned cache lines inclusive of __p
+     for (size_t __i = 0; __i < __nbytes; __i += __line_size) {
+       asm volatile("prefetch.global.L2::evict_last [%0];" ::"l"(__p + __i) :);
+     }))
+  // clang-format on
+}
+
+template <typename _Shape>
+_CCCL_HOST_DEVICE_API inline void apply_access_property(
+  [[maybe_unused]] const volatile void* __ptr,
+  [[maybe_unused]] _Shape __shape,
+  [[maybe_unused]] access_property::normal __prop) noexcept
+{
+  // clang-format off
+  NV_IF_TARGET(
+    NV_PROVIDES_SM_80,
+    (_CCCL_ASSERT(__ptr != nullptr, "null pointer");
+     if (!::cuda::device::is_address_from(__ptr, ::cuda::device::address_space::global))
+     {
+       return;
+     }
+     constexpr size_t __line_size = 128;
+     auto __p                     = reinterpret_cast<uint8_t*>(const_cast<void*>(__ptr));
+     auto __nbytes                = static_cast<size_t>(__shape);
+     // Apply to all 128 bytes aligned cache lines inclusive of __p
+     for (size_t __i = 0; __i < __nbytes; __i += __line_size) {
+       asm volatile("prefetch.global.L2::evict_normal [%0];" ::"l"(__p + __i) :);
+     }))
+  // clang-format on
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___ANNOTATED_PTR_APPLY_ACCESS_PROPERTY_H

cccl_upstream/libcudacxx/include/cuda/__annotated_ptr/associate_access_property.h

@@ -0,0 +1,127 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___ANNOTATED_PTR_ASSOCIATE_ACCESS_PROPERTY_H
+#define _CUDA___ANNOTATED_PTR_ASSOCIATE_ACCESS_PROPERTY_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__annotated_ptr/access_property.h>
+#include <cuda/__memory/address_space.h>
+#include <cuda/std/__type_traits/always_false.h>
+#include <cuda/std/__type_traits/is_one_of.h>
+#include <cuda/std/__type_traits/is_same.h>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+//----------------------------------------------------------------------------------------------------------------------
+// Private access property methods
+
+template <typename _Property>
+inline constexpr bool __is_access_property_v =
+  ::cuda::std::__is_one_of_v<_Property,
+                             access_property::shared,
+                             access_property::global,
+                             access_property::normal,
+                             access_property::persisting,
+                             access_property::streaming,
+                             access_property>;
+
+template <typename _Property>
+inline constexpr bool __is_global_access_property_v =
+  ::cuda::std::__is_one_of_v<_Property,
+                             access_property::global,
+                             access_property::normal,
+                             access_property::persisting,
+                             access_property::streaming,
+                             access_property>;
+
+#if _CCCL_CUDA_COMPILATION()
+
+template <typename _Property>
+[[nodiscard]] _CCCL_DEVICE_API void* __associate_address_space(void* __ptr, [[maybe_unused]] _Property __prop)
+{
+  if constexpr (::cuda::std::is_same_v<_Property, access_property::shared>)
+  {
+    [[maybe_unused]] bool __b = ::cuda::device::is_address_from(__ptr, ::cuda::device::address_space::shared);
+    _CCCL_ASSERT(__b, "");
+    _CCCL_ASSUME(__b);
+  }
+  else if constexpr (__is_global_access_property_v<_Property>)
+  {
+    [[maybe_unused]] bool __b = ::cuda::device::is_address_from(__ptr, ::cuda::device::address_space::global);
+    _CCCL_ASSERT(__b, "");
+    _CCCL_ASSUME(__b);
+  }
+  else
+  {
+    static_assert(::cuda::std::__always_false_v<_Property>, "invalid access_property");
+  }
+  return __ptr;
+}
+
+_CCCL_DEVICE_API inline void* __associate_raw_descriptor(void* __ptr, [[maybe_unused]] uint64_t __prop)
+{
+  NV_IF_TARGET(NV_PROVIDES_SM_80, (return ::__nv_associate_access_property(__ptr, __prop);))
+  return __ptr;
+}
+
+template <typename _Property>
+[[nodiscard]] _CCCL_DEVICE_API void* __associate_descriptor(void* __ptr, _Property __prop)
+{
+  static_assert(__is_access_property_v<_Property>, "invalid cuda::access_property");
+  if constexpr (!::cuda::std::is_same_v<_Property, access_property::shared>)
+  {
+    [[maybe_unused]] auto __raw_prop = static_cast<uint64_t>(access_property{__prop});
+    return ::cuda::__associate_raw_descriptor(__ptr, __raw_prop);
+  }
+  return __ptr;
+}
+
+#endif // _CCCL_CUDA_COMPILATION()
+
+template <typename _Type, typename _Property>
+[[nodiscard]] _CCCL_HOST_DEVICE_API inline _Type* __associate(_Type* __ptr, [[maybe_unused]] _Property __prop) noexcept
+{
+  static_assert(__is_access_property_v<_Property>, "invalid cuda::access_property");
+  NV_IF_ELSE_TARGET(
+    NV_IS_DEVICE,
+    (auto __void_ptr       = const_cast<void*>(static_cast<const void*>(__ptr));
+     auto __associated_ptr = ::cuda::__associate_address_space(__void_ptr, __prop);
+     return static_cast<_Type*>(::cuda::__associate_descriptor(__associated_ptr, __prop));),
+    (return __ptr;))
+}
+
+//----------------------------------------------------------------------------------------------------------------------
+// Public access property methods
+
+template <typename _Tp, typename _Property>
+[[nodiscard]] _CCCL_HOST_DEVICE_API inline _Tp* associate_access_property(_Tp* __ptr, _Property __prop) noexcept
+{
+  static_assert(__is_access_property_v<_Property>, "invalid cuda::access_property");
+  return ::cuda::__associate(__ptr, __prop);
+}
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___ANNOTATED_PTR_ASSOCIATE_ACCESS_PROPERTY_H

cccl_upstream/libcudacxx/include/cuda/__annotated_ptr/createpolicy.h

@@ -0,0 +1,210 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA___ANNOTATED_PTR_CREATEPOLICY_H
+#define _CUDA___ANNOTATED_PTR_CREATEPOLICY_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__memory/address_space.h>
+#include <cuda/std/cstddef>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA
+
+enum class __l2_evict_t : uint32_t
+{
+  _L2_Evict_Unchanged     = 0, // called "_L2_Evict_Normal" at lower level
+  _L2_Evict_First         = 1,
+  _L2_Evict_Last          = 2,
+  _L2_Evict_Normal_Demote = 3
+};
+
+/***********************************************************************************************************************
+ * PTX MAPPING
+ **********************************************************************************************************************/
+
+#if _CCCL_CUDA_COMPILATION()
+
+template <typename = void>
+[[nodiscard]] _CCCL_CONST _CCCL_DEVICE_API uint64_t __createpolicy_range_ptx(
+  __l2_evict_t __primary, __l2_evict_t __secondary, size_t __gmem_ptr, uint32_t __primary_size, uint32_t __total_size)
+{
+  uint64_t __policy;
+  if (__secondary == __l2_evict_t::_L2_Evict_Unchanged)
+  {
+    if (__primary == __l2_evict_t::_L2_Evict_Last)
+    {
+      asm("createpolicy.range.global.L2::evict_last.b64 %0, [%1], %2, %3;"
+          : "=l"(__policy)
+          : "l"(__gmem_ptr), "r"(__primary_size), "r"(__total_size));
+    }
+    else if (__primary == __l2_evict_t::_L2_Evict_Normal_Demote)
+    {
+      asm("createpolicy.range.global.L2::evict_normal.b64 %0, [%1], %2, %3;"
+          : "=l"(__policy)
+          : "l"(__gmem_ptr), "r"(__primary_size), "r"(__total_size));
+    }
+    else if (__primary == __l2_evict_t::_L2_Evict_First)
+    {
+      asm("createpolicy.range.global.L2::evict_first.b64 %0, [%1], %2, %3;"
+          : "=l"(__policy)
+          : "l"(__gmem_ptr), "r"(__primary_size), "r"(__total_size));
+    }
+    else if (__primary == __l2_evict_t::_L2_Evict_Unchanged)
+    {
+      asm("createpolicy.range.global.L2::evict_unchanged.b64 %0, [%1], %2, %3;"
+          : "=l"(__policy)
+          : "l"(__gmem_ptr), "r"(__primary_size), "r"(__total_size));
+    }
+    else
+    {
+      _CCCL_UNREACHABLE();
+    }
+  }
+  else // __secondary == _L2_Evict_First
+  {
+    if (__primary == __l2_evict_t::_L2_Evict_Last)
+    {
+      asm("createpolicy.range.global.L2::evict_last.L2::evict_first.b64 %0, [%1], %2, %3;"
+          : "=l"(__policy)
+          : "l"(__gmem_ptr), "r"(__primary_size), "r"(__total_size));
+    }
+    else if (__primary == __l2_evict_t::_L2_Evict_Normal_Demote)
+    {
+      asm("createpolicy.range.global.L2::evict_normal.L2::evict_first.b64 %0, [%1], %2, %3;"
+          : "=l"(__policy)
+          : "l"(__gmem_ptr), "r"(__primary_size), "r"(__total_size));
+    }
+    else if (__primary == __l2_evict_t::_L2_Evict_First)
+    {
+      asm("createpolicy.range.global.L2::evict_first.L2::evict_first.b64 %0, [%1], %2, %3;"
+          : "=l"(__policy)
+          : "l"(__gmem_ptr), "r"(__primary_size), "r"(__total_size));
+    }
+    else if (__primary == __l2_evict_t::_L2_Evict_Unchanged)
+    {
+      asm("createpolicy.range.global.L2::evict_unchanged.L2::evict_first.b64 %0, [%1], %2, %3;"
+          : "=l"(__policy)
+          : "l"(__gmem_ptr), "r"(__primary_size), "r"(__total_size));
+    }
+    else
+    {
+      _CCCL_UNREACHABLE();
+    }
+  }
+  return __policy;
+}
+
+template <typename = void>
+[[nodiscard]] _CCCL_CONST _CCCL_DEVICE_API uint64_t
+__createpolicy_fraction_ptx(__l2_evict_t __primary, __l2_evict_t __secondary, float __fraction)
+{
+  uint64_t __policy;
+  if (__secondary == __l2_evict_t::_L2_Evict_Unchanged)
+  {
+    if (__primary == __l2_evict_t::_L2_Evict_Last)
+    {
+      asm("createpolicy.fractional.L2::evict_last.b64 %0, %1;" : "=l"(__policy) : "f"(__fraction));
+    }
+    else if (__primary == __l2_evict_t::_L2_Evict_Normal_Demote)
+    {
+      asm("createpolicy.fractional.L2::evict_normal.b64 %0, %1;" : "=l"(__policy) : "f"(__fraction));
+    }
+    else if (__primary == __l2_evict_t::_L2_Evict_First)
+    {
+      asm("createpolicy.fractional.L2::evict_first.b64 %0, %1;" : "=l"(__policy) : "f"(__fraction));
+    }
+    else if (__primary == __l2_evict_t::_L2_Evict_Unchanged)
+    {
+      asm("createpolicy.fractional.L2::evict_unchanged.b64 %0, %1;" : "=l"(__policy) : "f"(__fraction));
+    }
+    else
+    {
+      _CCCL_UNREACHABLE();
+    }
+  }
+  else // __secondary == _L2_Evict_First
+  {
+    if (__primary == __l2_evict_t::_L2_Evict_Last)
+    {
+      asm("createpolicy.fractional.L2::evict_last.L2::evict_first.b64 %0, %1;" : "=l"(__policy) : "f"(__fraction));
+    }
+    else if (__primary == __l2_evict_t::_L2_Evict_Normal_Demote)
+    {
+      asm("createpolicy.fractional.L2::evict_normal.L2::evict_first.b64 %0, %1;" : "=l"(__policy) : "f"(__fraction));
+    }
+    else if (__primary == __l2_evict_t::_L2_Evict_First)
+    {
+      asm("createpolicy.fractional.L2::evict_first.L2::evict_first.b64 %0, %1;" : "=l"(__policy) : "f"(__fraction));
+    }
+    else if (__primary == __l2_evict_t::_L2_Evict_Unchanged)
+    {
+      asm("createpolicy.fractional.L2::evict_unchanged.L2::evict_first.b64 %0, %1;" : "=l"(__policy) : "f"(__fraction));
+    }
+    else
+    {
+      _CCCL_UNREACHABLE();
+    }
+  }
+  return __policy;
+}
+
+/***********************************************************************************************************************
+ * C++ API
+ **********************************************************************************************************************/
+
+extern "C" _CCCL_DEVICE void __createpolicy_is_not_supported_before_SM_80();
+
+template <typename T = void>
+[[nodiscard]] _CCCL_CONST _CCCL_DEVICE_API uint64_t __createpolicy_range(
+  __l2_evict_t __primary, __l2_evict_t __secondary, const void* __ptr, uint32_t __primary_size, uint32_t __total_size)
+{
+  _CCCL_ASSERT(::cuda::device::is_address_from(__ptr, ::cuda::device::address_space::global), "ptr must be global");
+  _CCCL_ASSERT(__primary_size > 0, "primary_size  must be greater than zero");
+  _CCCL_ASSERT(__primary_size <= __total_size, "primary_size must be less than or equal to total_size");
+  _CCCL_ASSERT(__secondary == __l2_evict_t::_L2_Evict_First || __secondary == __l2_evict_t::_L2_Evict_Unchanged,
+               "secondary policy must be evict_first or evict_unchanged");
+  [[maybe_unused]] auto __gmem_ptr = ::__cvta_generic_to_global(__ptr);
+  NV_IF_ELSE_TARGET(
+    NV_PROVIDES_SM_80,
+    (return ::cuda::__createpolicy_range_ptx(__primary, __secondary, __gmem_ptr, __primary_size, __total_size);),
+    (::cuda::__createpolicy_is_not_supported_before_SM_80(); return 0;))
+}
+
+template <typename T = void>
+[[nodiscard]] _CCCL_CONST _CCCL_DEVICE_API uint64_t
+__createpolicy_fraction(__l2_evict_t __primary, __l2_evict_t __secondary, float __fraction = 1.0f)
+{
+  _CCCL_ASSERT(__fraction > 0.0f && __fraction <= 1.0f, "fraction must be between 0.0f and 1.0f");
+  _CCCL_ASSERT(__secondary == __l2_evict_t::_L2_Evict_First || __secondary == __l2_evict_t::_L2_Evict_Unchanged,
+               "secondary policy must be evict_first or evict_unchanged");
+  NV_IF_ELSE_TARGET(NV_PROVIDES_SM_80,
+                    (return ::cuda::__createpolicy_fraction_ptx(__primary, __secondary, __fraction);),
+                    (::cuda::__createpolicy_is_not_supported_before_SM_80(); return 0;))
+}
+
+#endif // _CCCL_CUDA_COMPILATION()
+
+_CCCL_END_NAMESPACE_CUDA
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA___ANNOTATED_PTR_CREATEPOLICY_H