[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/libcudacxx/codegen/generators/compare_and_swap.h

@@ -0,0 +1,201 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef COMPARED_AND_SWAP_H
+#define COMPARED_AND_SWAP_H
+
+#include <format>
+#include <string>
+
+#include "definitions.h"
+
+inline void FormatCompareAndSwap(std::ostream& out)
+{
+  out << R"XXX(
+template <class _Fn, class _Sco>
+static inline _CCCL_DEVICE bool __cuda_atomic_compare_swap_memory_order_dispatch(_Fn& __cuda_cas, int __success_memorder, int __failure_memorder, _Sco) {
+  bool __res = false;
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (
+      switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) {
+        case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __atomic_cuda_seq_cst{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __res = __cuda_cas(__atomic_cuda_acquire{}); break;
+        case __ATOMIC_ACQ_REL: __res = __cuda_cas(__atomic_cuda_acq_rel{}); break;
+        case __ATOMIC_RELEASE: __res = __cuda_cas(__atomic_cuda_release{}); break;
+        case __ATOMIC_RELAXED: __res = __cuda_cas(__atomic_cuda_relaxed{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    ),
+    NV_IS_DEVICE, (
+      switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) {
+        case __ATOMIC_SEQ_CST: [[fallthrough]];
+        case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __res = __cuda_cas(__atomic_cuda_volatile{}); __cuda_atomic_membar(_Sco{}); break;
+        case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __res = __cuda_cas(__atomic_cuda_volatile{}); break;
+        case __ATOMIC_RELAXED: __res = __cuda_cas(__atomic_cuda_volatile{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    )
+  )
+  return __res;
+}
+)XXX";
+
+  // Argument ID Reference
+  // 0 - Operand Type
+  // 1 - Operand Size
+  // 2 - Type Constraint
+  // 3 - Memory Order
+  // 4 - Memory Order function tag
+  // 5 - Scope Constraint
+  // 6 - Scope function tag
+  constexpr auto asm_intrinsic_format_128 = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange(
+  _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, {4}, __atomic_cuda_operand_{0}{1}, {6})
+{{
+  static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840");
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_90, (),
+    NV_ANY_TARGET, (__atomic_cas_128b_unsupported_before_SM_90();)
+  )
+  asm volatile(R"YYY(
+    {{
+      .reg .b128 _d;
+      .reg .b128 _v;
+      mov.b128 _d, {{%3, %4}};
+      mov.b128 _v, {{%5, %6}};
+      atom.cas{3}{5}.b128 _d,[%2],_d,_v;
+      mov.b128 {{%0, %1}}, _d;
+    }}
+  )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; }})XXX";
+
+  constexpr auto asm_intrinsic_format = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange(
+  _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, {4}, __atomic_cuda_operand_{0}{1}, {6})
+{{ asm volatile("atom.cas{3}{5}.{0}{1} %0,[%1],%2,%3;" : "={2}"(__dst) : "l"(__ptr), "{2}"(__cmp), "{2}"(__op) : "memory"); return __dst == __cmp; }})XXX";
+
+  constexpr Operand supported_types[] = {
+    Operand::Bit,
+  };
+
+  constexpr size_t supported_sizes[] = {
+    32,
+    64,
+    128,
+  };
+
+  constexpr Semantic supported_semantics[] = {
+    Semantic::Acquire,
+    Semantic::Relaxed,
+    Semantic::Release,
+    Semantic::Acq_Rel,
+    Semantic::Volatile,
+  };
+
+  constexpr Scope supported_scopes[] = {
+    Scope::CTA,
+    Scope::Cluster,
+    Scope::GPU,
+    Scope::System,
+  };
+
+  for (auto size : supported_sizes)
+  {
+    for (auto type : supported_types)
+    {
+      for (auto sem : supported_semantics)
+      {
+        for (auto sco : supported_scopes)
+        {
+          if (size == 2 && type != Operand::Bit)
+          {
+            continue;
+          }
+          if (size == 128 && type != Operand::Bit)
+          {
+            continue;
+          }
+
+          if (size == 128)
+          {
+            out << std::format(
+              asm_intrinsic_format_128,
+              operand(type),
+              size,
+              constraints(type, size),
+              semantic(sem),
+              semantic_tag(sem),
+              scope(sco),
+              scope_tag(sco));
+          }
+          else
+          {
+            out << std::format(
+              asm_intrinsic_format,
+              operand(type),
+              size,
+              constraints(type, size),
+              semantic(sem),
+              semantic_tag(sem),
+              scope(sco),
+              scope_tag(sco));
+          }
+        }
+      }
+    }
+  }
+
+  out << "\n"
+      << R"XXX(
+template <typename _Type, typename _Tag, typename _Sco>
+struct __cuda_atomic_bind_compare_exchange {
+  _Type* __ptr;
+  _Type* __exp;
+  _Type* __des;
+
+  template <typename _Atomic_Memorder>
+  inline _CCCL_DEVICE bool operator()(_Atomic_Memorder) {
+    return __cuda_atomic_compare_exchange(__ptr, *__exp, *__exp, *__des, _Atomic_Memorder{}, _Tag{}, _Sco{});
+  }
+};
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE bool __atomic_compare_exchange_cuda(_Type* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr);
+  __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp);
+  __proxy_t* __des_proxy  = reinterpret_cast<__proxy_t*>(&__des);
+  bool __res = false;
+  if (__cuda_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;}
+  __cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy};
+  return __cuda_atomic_compare_swap_memory_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{});
+}
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE bool __atomic_compare_exchange_cuda(_Type volatile* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr));
+  __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp);
+  __proxy_t* __des_proxy  = reinterpret_cast<__proxy_t*>(&__des);
+  bool __res = false;
+  if (__cuda_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;}
+  __cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy};
+  return __cuda_atomic_compare_swap_memory_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{});
+}
+)XXX";
+}
+
+#endif // COMPARED_AND_SWAP_H

cccl_upstream/libcudacxx/codegen/generators/definitions.h

@@ -0,0 +1,192 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef DEFINITIONS_H
+#define DEFINITIONS_H
+
+#include <format>
+#include <map>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+enum class Mmio
+{
+  Disabled,
+  Enabled,
+};
+
+inline std::string mmio(Mmio m)
+{
+  static const char* mmio_map[]{
+    "",
+    ".mmio",
+  };
+  return mmio_map[std::underlying_type_t<Mmio>(m)];
+}
+
+inline std::string mmio_tag(Mmio m)
+{
+  static const char* mmio_map[]{
+    "__atomic_cuda_mmio_disable",
+    "__atomic_cuda_mmio_enable",
+  };
+  return mmio_map[std::underlying_type_t<Mmio>(m)];
+}
+
+enum class Operand
+{
+  Floating,
+  Unsigned,
+  Signed,
+  Bit,
+};
+
+inline std::string operand(Operand op)
+{
+  static std::map op_map = {
+    std::pair{Operand::Floating, "f"},
+    std::pair{Operand::Unsigned, "u"},
+    std::pair{Operand::Signed, "s"},
+    std::pair{Operand::Bit, "b"},
+  };
+  return op_map[op];
+}
+
+inline std::string operand_proxy_type(Operand op, size_t sz)
+{
+  if (op == Operand::Floating)
+  {
+    if (sz == 32)
+    {
+      return {"float"};
+    }
+    else
+    {
+      return {"double"};
+    }
+  }
+  else if (op == Operand::Signed)
+  {
+    return std::format("int{}_t", sz);
+  }
+  // Binary and unsigned can be the same proxy_type
+  return std::format("uint{}_t", sz);
+}
+
+inline std::string constraints(Operand op, size_t sz)
+{
+  static std::map constraint_map = {
+    std::pair{32,
+              std::map{
+                std::pair{Operand::Bit, "r"},
+                std::pair{Operand::Unsigned, "r"},
+                std::pair{Operand::Signed, "r"},
+                std::pair{Operand::Floating, "f"},
+              }},
+    std::pair{64,
+              std::map{
+                std::pair{Operand::Bit, "l"},
+                std::pair{Operand::Unsigned, "l"},
+                std::pair{Operand::Signed, "l"},
+                std::pair{Operand::Floating, "d"},
+              }},
+    std::pair{128,
+              std::map{
+                std::pair{Operand::Bit, "l"},
+                std::pair{Operand::Unsigned, "l"},
+                std::pair{Operand::Signed, "l"},
+                std::pair{Operand::Floating, "d"},
+              }},
+  };
+
+  if (sz == 16)
+  {
+    return {"h"};
+  }
+  else
+  {
+    return constraint_map[sz][op];
+  }
+}
+
+enum class Semantic
+{
+  Relaxed,
+  Release,
+  Acquire,
+  Acq_Rel,
+  Seq_Cst,
+  Volatile,
+};
+
+inline std::string semantic(Semantic sem)
+{
+  static std::map sem_map = {
+    std::pair{Semantic::Relaxed, ".relaxed"},
+    std::pair{Semantic::Release, ".release"},
+    std::pair{Semantic::Acquire, ".acquire"},
+    std::pair{Semantic::Acq_Rel, ".acq_rel"},
+    std::pair{Semantic::Seq_Cst, ".sc"},
+    std::pair{Semantic::Volatile, ""},
+  };
+  return sem_map[sem];
+}
+
+inline std::string semantic_tag(Semantic sem)
+{
+  static std::map sem_map = {
+    std::pair{Semantic::Relaxed, "__atomic_cuda_relaxed"},
+    std::pair{Semantic::Release, "__atomic_cuda_release"},
+    std::pair{Semantic::Acquire, "__atomic_cuda_acquire"},
+    std::pair{Semantic::Acq_Rel, "__atomic_cuda_acq_rel"},
+    std::pair{Semantic::Seq_Cst, "__atomic_cuda_seq_cst"},
+    std::pair{Semantic::Volatile, "__atomic_cuda_volatile"},
+  };
+  return sem_map[sem];
+}
+
+enum class Scope
+{
+  Thread,
+  Warp,
+  CTA,
+  Cluster,
+  GPU,
+  System,
+};
+
+inline std::string scope(Scope sco)
+{
+  static std::map sco_map = {
+    std::pair{Scope::Thread, ""},
+    std::pair{Scope::Warp, ""},
+    std::pair{Scope::CTA, ".cta"},
+    std::pair{Scope::Cluster, ".cluster"},
+    std::pair{Scope::GPU, ".gpu"},
+    std::pair{Scope::System, ".sys"},
+  };
+  return sco_map[sco];
+}
+
+inline std::string scope_tag(Scope sco)
+{
+  static std::map sco_map = {
+    std::pair{Scope::Thread, "__thread_scope_thread_tag"},
+    std::pair{Scope::Warp, ""},
+    std::pair{Scope::CTA, "__thread_scope_block_tag"},
+    std::pair{Scope::Cluster, "__thread_scope_cluster_tag"},
+    std::pair{Scope::GPU, "__thread_scope_device_tag"},
+    std::pair{Scope::System, "__thread_scope_system_tag"},
+  };
+  return sco_map[sco];
+}
+
+#endif // DEFINITIONS_H

cccl_upstream/libcudacxx/codegen/generators/exchange.h

@@ -0,0 +1,197 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef EXCHANGE_H
+#define EXCHANGE_H
+
+#include <format>
+#include <string>
+
+#include "definitions.h"
+
+inline void FormatExchange(std::ostream& out)
+{
+  out << R"XXX(
+template <class _Fn, class _Sco>
+static inline _CCCL_DEVICE void __cuda_atomic_exchange_memory_order_dispatch(_Fn& __cuda_exch, int __memorder, _Sco) {
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __atomic_cuda_seq_cst{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __cuda_exch(__atomic_cuda_acquire{}); break;
+        case __ATOMIC_ACQ_REL: __cuda_exch(__atomic_cuda_acq_rel{}); break;
+        case __ATOMIC_RELEASE: __cuda_exch(__atomic_cuda_release{}); break;
+        case __ATOMIC_RELAXED: __cuda_exch(__atomic_cuda_relaxed{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    ),
+    NV_IS_DEVICE, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: [[fallthrough]];
+        case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __cuda_exch(__atomic_cuda_volatile{}); __cuda_atomic_membar(_Sco{}); break;
+        case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __cuda_exch(__atomic_cuda_volatile{}); break;
+        case __ATOMIC_RELAXED: __cuda_exch(__atomic_cuda_volatile{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    )
+  )
+}
+)XXX";
+
+  // Argument ID Reference
+  // 0 - Operand Type
+  // 1 - Operand Size
+  // 2 - Type Constraint
+  // 3 - Memory Order
+  // 4 - Memory Order function tag
+  // 5 - Scope Constraint
+  // 6 - Scope function tag
+  constexpr auto asm_intrinsic_format_128 = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_exchange(
+  _Type* __ptr, _Type& __old, _Type __new, {4}, __atomic_cuda_operand_{0}{1}, {6})
+{{
+  static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840");
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_90, (),
+    NV_ANY_TARGET, (__atomic_exchange_128b_unsupported_before_SM_90();)
+  )
+  asm volatile(R"YYY(
+    {{
+      .reg .b128 _d;
+      .reg .b128 _v;
+      mov.b128 _v, {{%3, %4}};
+      atom.exch{3}{5}.b128 _d,[%2],_v;
+      mov.b128 {{%0, %1}}, _d;
+    }}
+  )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory");
+}})XXX";
+
+  constexpr auto asm_intrinsic_format = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_exchange(
+  _Type* __ptr, _Type& __old, _Type __new, {4}, __atomic_cuda_operand_{0}{1}, {6})
+{{ asm volatile("atom.exch{3}{5}.{0}{1} %0,[%1],%2;" : "={2}"(__old) : "l"(__ptr), "{2}"(__new) : "memory"); }})XXX";
+
+  constexpr Operand supported_types[] = {
+    Operand::Bit,
+  };
+
+  constexpr size_t supported_sizes[] = {
+    32,
+    64,
+    128,
+  };
+
+  constexpr Semantic supported_semantics[] = {
+    Semantic::Acquire,
+    Semantic::Relaxed,
+    Semantic::Release,
+    Semantic::Acq_Rel,
+    Semantic::Volatile,
+  };
+
+  constexpr Scope supported_scopes[] = {
+    Scope::CTA,
+    Scope::Cluster,
+    Scope::GPU,
+    Scope::System,
+  };
+
+  for (auto size : supported_sizes)
+  {
+    for (auto type : supported_types)
+    {
+      for (auto sem : supported_semantics)
+      {
+        for (auto sco : supported_scopes)
+        {
+          if (size == 2 && type != Operand::Bit)
+          {
+            continue;
+          }
+          if (size == 128 && type != Operand::Bit)
+          {
+            continue;
+          }
+
+          if (size == 128)
+          {
+            out << std::format(
+              asm_intrinsic_format_128,
+              operand(type),
+              size,
+              constraints(type, size),
+              semantic(sem),
+              semantic_tag(sem),
+              scope(sco),
+              scope_tag(sco));
+          }
+          else
+          {
+            out << std::format(
+              asm_intrinsic_format,
+              operand(type),
+              size,
+              constraints(type, size),
+              semantic(sem),
+              semantic_tag(sem),
+              scope(sco),
+              scope_tag(sco));
+          }
+        }
+      }
+    }
+  }
+
+  out << "\n"
+      << R"XXX(
+template <typename _Type, typename _Tag, typename _Sco>
+struct __cuda_atomic_bind_exchange {
+  _Type* __ptr;
+  _Type* __old;
+  _Type* __new;
+
+  template <typename _Atomic_Memorder>
+  inline _CCCL_DEVICE void operator()(_Atomic_Memorder) {
+    __cuda_atomic_exchange(__ptr, *__old, *__new, _Atomic_Memorder{}, _Tag{}, _Sco{});
+  }
+};
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE void __atomic_exchange_cuda(_Type* __ptr, _Type& __old, _Type __new, int __memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr);
+  __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old);
+  __proxy_t* __new_proxy  = reinterpret_cast<__proxy_t*>(&__new);
+  if(__cuda_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}}
+  __cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy};
+  __cuda_atomic_exchange_memory_order_dispatch(__bound_swap, __memorder, _Sco{});
+}
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE void __atomic_exchange_cuda(_Type volatile* __ptr, _Type& __old, _Type __new, int __memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr));
+  __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old);
+  __proxy_t* __new_proxy  = reinterpret_cast<__proxy_t*>(&__new);
+  if(__cuda_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}}
+  __cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy};
+  __cuda_atomic_exchange_memory_order_dispatch(__bound_swap, __memorder, _Sco{});
+}
+)XXX";
+}
+
+#endif // EXCHANGE_H

cccl_upstream/libcudacxx/codegen/generators/fence.h

@@ -0,0 +1,110 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FENCE_H
+#define FENCE_H
+
+#include <format>
+#include <string>
+
+#include "definitions.h"
+
+inline std::string membar_scope(Scope sco)
+{
+  static std::map scope_map{
+    std::pair{Scope::GPU, ".gl"},
+    std::pair{Scope::System, ".sys"},
+    std::pair{Scope::CTA, ".cta"},
+  };
+
+  return scope_map[sco];
+}
+
+inline void FormatFence(std::ostream& out)
+{
+  // Argument ID Reference
+  // 0 - Membar scope tag
+  // 1 - Membar scope
+  constexpr auto intrinsic_membar = R"XXX(
+static inline _CCCL_DEVICE void __cuda_atomic_membar({0})
+{{ asm volatile("membar{1};" ::: "memory"); }})XXX";
+
+  const std::map membar_scopes{
+    std::pair{Scope::GPU, ".gl"},
+    std::pair{Scope::System, ".sys"},
+    std::pair{Scope::CTA, ".cta"},
+  };
+
+  for (const auto& sco : membar_scopes)
+  {
+    out << std::format(intrinsic_membar, scope_tag(sco.first), sco.second);
+  }
+
+  // Argument ID Reference
+  // 0 - Fence scope tag
+  // 1 - Fence scope
+  // 2 - Fence order tag
+  // 3 - Fence order
+  constexpr auto intrinsic_fence = R"XXX(
+static inline _CCCL_DEVICE void __cuda_atomic_fence({0}, {2})
+{{ asm volatile("fence{1}{3};" ::: "memory"); }})XXX";
+
+  const Scope fence_scopes[] = {
+    Scope::CTA,
+    Scope::Cluster,
+    Scope::GPU,
+    Scope::System,
+  };
+
+  const Semantic fence_semantics[] = {
+    Semantic::Acq_Rel,
+    Semantic::Seq_Cst,
+  };
+
+  for (const auto& sco : fence_scopes)
+  {
+    for (const auto& sem : fence_semantics)
+    {
+      out << std::format(intrinsic_fence, scope_tag(sco), semantic(sem), semantic_tag(sem), scope(sco));
+    }
+  }
+  out << "\n"
+      << R"XXX(
+template <typename _Sco>
+static inline _CCCL_DEVICE void __atomic_thread_fence_cuda(int __memorder, _Sco) {
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __atomic_cuda_seq_cst{}); break;
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: [[fallthrough]];
+        case __ATOMIC_ACQ_REL: [[fallthrough]];
+        case __ATOMIC_RELEASE: __cuda_atomic_fence(_Sco{}, __atomic_cuda_acq_rel{}); break;
+        case __ATOMIC_RELAXED: break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    ),
+    NV_IS_DEVICE, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: [[fallthrough]];
+        case __ATOMIC_ACQ_REL: [[fallthrough]];
+        case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); break;
+        case __ATOMIC_RELAXED: break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    )
+  )
+}
+)XXX";
+}
+
+#endif // FENCE_H

cccl_upstream/libcudacxx/codegen/generators/fetch_ops.h

@@ -0,0 +1,219 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FETCH_OPS_H
+#define FETCH_OPS_H
+
+#include <array>
+#include <format>
+#include <string>
+
+#include "definitions.h"
+
+inline std::string fetch_op_skip_v(std::string fetch_op)
+{
+  if (fetch_op == "add")
+  {
+    return "constexpr auto __skip_v = __atomic_ptr_skip_t<_Type>::__skip;";
+  }
+  return "constexpr auto __skip_v = 1;";
+}
+
+inline void FormatFetchOps(std::ostream& out)
+{
+  const std::vector arithmetic_types = {
+    Operand::Floating,
+    Operand::Unsigned,
+    Operand::Signed,
+  };
+
+  const std::vector minmax_types = {
+    Operand::Unsigned,
+    Operand::Signed,
+  };
+
+  const std::vector bitwise_types = {Operand::Bit};
+
+  const std::map op_support_map{
+    std::pair{std::string{"add"}, std::pair{arithmetic_types, std::string{"arithmetic"}}},
+    std::pair{std::string{"min"}, std::pair{minmax_types, std::string{"minmax"}}},
+    std::pair{std::string{"max"}, std::pair{minmax_types, std::string{"minmax"}}},
+    std::pair{std::string{"or"}, std::pair{bitwise_types, std::string{"bitwise"}}},
+    std::pair{std::string{"xor"}, std::pair{bitwise_types, std::string{"bitwise"}}},
+    std::pair{std::string{"and"}, std::pair{bitwise_types, std::string{"bitwise"}}},
+  };
+
+  // Memory order dispatcher
+  out << R"XXX(
+template <class _Fn, class _Sco>
+static inline _CCCL_DEVICE void __cuda_atomic_fetch_memory_order_dispatch(_Fn& __cuda_fetch, int __memorder, _Sco) {
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __atomic_cuda_seq_cst{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __cuda_fetch(__atomic_cuda_acquire{}); break;
+        case __ATOMIC_ACQ_REL: __cuda_fetch(__atomic_cuda_acq_rel{}); break;
+        case __ATOMIC_RELEASE: __cuda_fetch(__atomic_cuda_release{}); break;
+        case __ATOMIC_RELAXED: __cuda_fetch(__atomic_cuda_relaxed{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    ),
+    NV_IS_DEVICE, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: [[fallthrough]];
+        case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __cuda_fetch(__atomic_cuda_volatile{}); __cuda_atomic_membar(_Sco{}); break;
+        case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __cuda_fetch(__atomic_cuda_volatile{}); break;
+        case __ATOMIC_RELAXED: __cuda_fetch(__atomic_cuda_volatile{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    )
+  )
+}
+)XXX";
+
+  // Argument ID Reference
+  // 0 - Atomic Operation
+  // 1 - Operand Type
+  // 2 - Operand Size
+  // 3 - Type Constraint
+  // 4 - Memory Order
+  // 5 - Memory Order function tag
+  // 6 - Scope Constraint
+  // 7 - Scope function tag
+  constexpr auto asm_intrinsic_format = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_fetch_{0}(
+  _Type* __ptr, _Type& __dst, _Type __op, {5}, __atomic_cuda_operand_{1}{2}, {7})
+{{ asm volatile("atom.{0}{4}{6}.{1}{2} %0,[%1],%2;" : "={3}"(__dst) : "l"(__ptr), "{3}"(__op) : "memory"); }})XXX";
+
+  // 0 - Atomic Operation
+  // 1 - Operand type constraint
+  // 2 - Pointer op skip_v
+  constexpr auto fetch_bind_invoke = R"XXX(
+template <typename _Type, typename _Tag, typename _Sco>
+struct __cuda_atomic_bind_fetch_{0} {{
+  _Type* __ptr;
+  _Type* __dst;
+  _Type* __op;
+
+  template <typename _Atomic_Memorder>
+  inline _CCCL_DEVICE void operator()(_Atomic_Memorder) {{
+    __cuda_atomic_fetch_{0}(__ptr, *__dst, *__op, _Atomic_Memorder{{}}, _Tag{{}}, _Sco{{}});
+  }}
+}};
+template <class _Type, class _Up, class _Sco, __atomic_enable_if_native_{1}<_Type> = 0>
+[[nodiscard]] static inline _CCCL_DEVICE _Type __atomic_fetch_{0}_cuda(_Type* __ptr, _Up __op, int __memorder, _Sco)
+{{
+  {2}
+  __op = __op * __skip_v;
+  using __proxy_t        = typename __atomic_cuda_deduce_{1}<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_{1}<_Type>::__tag;
+  _Type __dst{{}};
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr);
+  __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst);
+  __proxy_t* __op_proxy  = reinterpret_cast<__proxy_t*>(&__op);
+  if (__cuda_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}}
+  __cuda_atomic_bind_fetch_{0}<__proxy_t, __proxy_tag, _Sco> __bound_{0}{{__ptr_proxy, __dst_proxy, __op_proxy}};
+  __cuda_atomic_fetch_memory_order_dispatch(__bound_{0}, __memorder, _Sco{{}});
+  return __dst;
+}}
+template <class _Type, class _Up, class _Sco, __atomic_enable_if_native_{1}<_Type> = 0>
+[[nodiscard]] static inline _CCCL_DEVICE _Type __atomic_fetch_{0}_cuda(_Type volatile* __ptr, _Up __op, int __memorder, _Sco)
+{{
+  {2}
+  __op = __op * __skip_v;
+  using __proxy_t        = typename __atomic_cuda_deduce_{1}<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_{1}<_Type>::__tag;
+  _Type __dst{{}};
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr));
+  __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst);
+  __proxy_t* __op_proxy  = reinterpret_cast<__proxy_t*>(&__op);
+  if (__cuda_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}}
+  __cuda_atomic_bind_fetch_{0}<__proxy_t, __proxy_tag, _Sco> __bound_{0}{{__ptr_proxy, __dst_proxy, __op_proxy}};
+  __cuda_atomic_fetch_memory_order_dispatch(__bound_{0}, __memorder, _Sco{{}});
+  return __dst;
+}}
+)XXX";
+
+  constexpr size_t supported_sizes[] = {
+    32,
+    64,
+  };
+
+  constexpr Semantic supported_semantics[] = {
+    Semantic::Acquire,
+    Semantic::Relaxed,
+    Semantic::Release,
+    Semantic::Acq_Rel,
+    Semantic::Volatile,
+  };
+
+  constexpr Scope supported_scopes[] = {
+    Scope::CTA,
+    Scope::Cluster,
+    Scope::GPU,
+    Scope::System,
+  };
+
+  for (auto& op_kp : op_support_map)
+  {
+    const auto& op_name    = op_kp.first;
+    const auto& op_type_kp = op_kp.second;
+    const auto& type_list  = op_type_kp.first;
+    const auto& deduction  = op_type_kp.second;
+    for (auto type : type_list)
+    {
+      for (auto size : supported_sizes)
+      {
+        const std::string proxy_type = operand_proxy_type(type, size);
+        for (auto sco : supported_scopes)
+        {
+          for (auto sem : supported_semantics)
+          {
+            // There is no atom.add.s64
+            if (op_name == "add" && type == Operand::Signed && size == 64)
+            {
+              continue;
+            }
+            out << std::format(
+              asm_intrinsic_format,
+              /* 0 */ op_name,
+              /* 1 */ operand(type),
+              /* 2 */ size,
+              /* 3 */ constraints(type, size),
+              /* 4 */ semantic(sem),
+              /* 5 */ semantic_tag(sem),
+              /* 6 */ scope(sco),
+              /* 7 */ scope_tag(sco));
+          }
+        }
+      }
+    }
+    out << "\n" << std::format(fetch_bind_invoke, op_name, deduction, fetch_op_skip_v(op_name));
+  }
+
+  out << R"XXX(
+template <class _Type, class _Up, class _Sco>
+[[nodiscard]] static inline _CCCL_DEVICE _Type __atomic_fetch_sub_cuda(_Type* __ptr, _Up __op, int __memorder, _Sco)
+{
+  return __atomic_fetch_add_cuda(__ptr, -__op, __memorder, _Sco{});
+}
+template <class _Type, class _Up, class _Sco>
+[[nodiscard]] static inline _CCCL_DEVICE _Type __atomic_fetch_sub_cuda(_Type volatile* __ptr, _Up __op, int __memorder, _Sco)
+{
+  return __atomic_fetch_add_cuda(__ptr, -__op, __memorder, _Sco{});
+}
+)XXX";
+}
+
+#endif // FETCH_OPS_H

cccl_upstream/libcudacxx/codegen/generators/header.h

@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HEADER_H
+#define HEADER_H
+
+#include <string>
+
+inline void FormatHeader(std::ostream& out)
+{
+  constexpr auto header = R"XXX(//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+// This is an autogenerated file, we want to ensure that it contains exactly the contents we want to generate
+// clang-format off
+
+#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_H
+#define _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/cassert>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/is_signed.h>
+#include <cuda/std/__type_traits/is_unsigned.h>
+
+#include <cuda/std/__atomic/scopes.h>
+#include <cuda/std/__atomic/order.h>
+#include <cuda/std/__atomic/functions/common.h>
+#include <cuda/std/__atomic/functions/cuda_ptx_generated_helper.h>
+#include <cuda/std/__atomic/functions/cuda_local.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+#if _CCCL_CUDA_COMPILATION()
+
+extern "C" _CCCL_DEVICE void __atomic_cas_128b_unsupported_before_SM_90();
+extern "C" _CCCL_DEVICE void __atomic_exchange_128b_unsupported_before_SM_90();
+extern "C" _CCCL_DEVICE void __atomic_ldst_128b_unsupported_before_SM_70();
+)XXX";
+
+  out << header;
+}
+
+inline void FormatTail(std::ostream& out)
+{
+  constexpr auto tail = R"XXX(
+#endif // _CCCL_CUDA_COMPILATION()
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_H
+
+// clang-format on
+)XXX";
+
+  out << tail;
+}
+
+#endif // HEADER_H

cccl_upstream/libcudacxx/codegen/generators/ld_st.h

@@ -0,0 +1,407 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LD_ST_H
+#define LD_ST_H
+
+#include <format>
+#include <string>
+
+#include "definitions.h"
+
+inline std::string semantic_ld_st(Semantic sem)
+{
+  static std::map sem_map = {
+    std::pair{Semantic::Relaxed, ".relaxed"},
+    std::pair{Semantic::Release, ".release"},
+    std::pair{Semantic::Acquire, ".acquire"},
+    std::pair{Semantic::Volatile, ".volatile"},
+  };
+  return sem_map[sem];
+}
+
+inline std::string scope_ld_st(Semantic sem, Scope sco)
+{
+  if (sem == Semantic::Volatile)
+  {
+    return "";
+  }
+  return scope(sco);
+}
+
+inline void FormatLoad(std::ostream& out)
+{
+  out << R"XXX(
+template <class _Fn, class _Sco>
+static inline _CCCL_DEVICE void __cuda_atomic_load_memory_order_dispatch(_Fn &__cuda_load, int __memorder, _Sco) {
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __atomic_cuda_seq_cst{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __cuda_load(__atomic_cuda_acquire{}); break;
+        case __ATOMIC_RELAXED: __cuda_load(__atomic_cuda_relaxed{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    ),
+    NV_IS_DEVICE, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: __cuda_atomic_membar(_Sco{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __cuda_load(__atomic_cuda_volatile{}); __cuda_atomic_membar(_Sco{}); break;
+        case __ATOMIC_RELAXED: __cuda_load(__atomic_cuda_volatile{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    )
+  )
+}
+)XXX";
+
+  // Argument ID Reference
+  // 0 - Operand Type
+  // 1 - Operand Size
+  // 2 - Constraint
+  // 3 - Memory order
+  // 4 - Memory order semantic
+  // 5 - Scope tag
+  // 6 - Scope semantic
+  // 7 - Mmio tag
+  // 8 - Mmio semantic
+  constexpr auto asm_intrinsic_format_128 = R"XXX(
+  template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_load(
+  const _Type* __ptr, _Type& __dst, {3}, __atomic_cuda_operand_{0}{1}, {5}, {7})
+{{
+  static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840");
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (),
+    NV_ANY_TARGET, (__atomic_ldst_128b_unsupported_before_SM_70();)
+  )
+  asm volatile(R"YYY(
+    {{
+      .reg .b128 _d;
+      ld{8}{4}{6}.b128 _d,[%2];
+      mov.b128 {{%0, %1}}, _d;
+    }}
+  )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory");
+}})XXX";
+  constexpr auto asm_intrinsic_format     = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_load(
+  const _Type* __ptr, _Type& __dst, {3}, __atomic_cuda_operand_{0}{1}, {5}, {7})
+{{ asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__dst) : "l"(__ptr) : "memory"); }})XXX";
+
+  constexpr size_t supported_sizes[] = {
+    16,
+    32,
+    64,
+    128,
+  };
+
+  constexpr Operand supported_types[] = {
+    Operand::Bit,
+    Operand::Floating,
+    Operand::Unsigned,
+    Operand::Signed,
+  };
+
+  constexpr Semantic supported_semantics[] = {
+    Semantic::Acquire,
+    Semantic::Relaxed,
+    Semantic::Volatile,
+  };
+
+  constexpr Scope supported_scopes[] = {
+    Scope::CTA,
+    Scope::Cluster,
+    Scope::GPU,
+    Scope::System,
+  };
+
+  constexpr Mmio mmio_states[] = {
+    Mmio::Disabled,
+    Mmio::Enabled,
+  };
+
+  for (auto size : supported_sizes)
+  {
+    for (auto type : supported_types)
+    {
+      for (auto sem : supported_semantics)
+      {
+        for (auto sco : supported_scopes)
+        {
+          for (auto mm : mmio_states)
+          {
+            if (size == 16 && type == Operand::Floating)
+            {
+              continue;
+            }
+            if (size == 128 && type != Operand::Bit)
+            {
+              continue;
+            }
+            if ((mm == Mmio::Enabled) && ((sco != Scope::System) || (sem != Semantic::Relaxed)))
+            {
+              continue;
+            }
+
+            if (size == 128)
+            {
+              out << std::format(
+                asm_intrinsic_format_128,
+                /* 0 */ operand(type),
+                /* 1 */ size,
+                /* 2 */ constraints(type, size),
+                /* 3 */ semantic_tag(sem),
+                /* 4 */ semantic_ld_st(sem),
+                /* 5 */ scope_tag(sco),
+                /* 6 */ scope_ld_st(sem, sco),
+                /* 7 */ mmio_tag(mm),
+                /* 8 */ mmio(mm));
+            }
+            else
+            {
+              out << std::format(
+                asm_intrinsic_format,
+                /* 0 */ operand(type),
+                /* 1 */ size,
+                /* 2 */ constraints(type, size),
+                /* 3 */ semantic_tag(sem),
+                /* 4 */ semantic_ld_st(sem),
+                /* 5 */ scope_tag(sco),
+                /* 6 */ scope_ld_st(sem, sco),
+                /* 7 */ mmio_tag(mm),
+                /* 8 */ mmio(mm));
+            }
+          }
+        }
+      }
+    }
+  }
+  out << "\n"
+      << R"XXX(
+template <typename _Type, typename _Tag, typename _Sco, typename _Mmio>
+struct __cuda_atomic_bind_load {
+  const _Type* __ptr;
+  _Type* __dst;
+
+  template <typename _Atomic_Memorder>
+  inline _CCCL_DEVICE void operator()(_Atomic_Memorder) {
+    __cuda_atomic_load(__ptr, *__dst, _Atomic_Memorder{}, _Tag{}, _Sco{}, _Mmio{});
+  }
+};
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE void __atomic_load_cuda(const _Type* __ptr, _Type& __dst, int __memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  const __proxy_t* __ptr_proxy = reinterpret_cast<const __proxy_t*>(__ptr);
+  __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst);
+  if (__cuda_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {{return;}}
+  __cuda_atomic_bind_load<__proxy_t, __proxy_tag, _Sco, __atomic_cuda_mmio_disable> __bound_load{__ptr_proxy, __dst_proxy};
+  __cuda_atomic_load_memory_order_dispatch(__bound_load, __memorder, _Sco{});
+}
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE void __atomic_load_cuda(const _Type volatile* __ptr, _Type& __dst, int __memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  const __proxy_t* __ptr_proxy = reinterpret_cast<const __proxy_t*>(const_cast<_Type*>(__ptr));
+  __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst);
+  if (__cuda_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {{return;}}
+  __cuda_atomic_bind_load<__proxy_t, __proxy_tag, _Sco, __atomic_cuda_mmio_disable> __bound_load{__ptr_proxy, __dst_proxy};
+  __cuda_atomic_load_memory_order_dispatch(__bound_load, __memorder, _Sco{});
+}
+)XXX";
+}
+
+inline void FormatStore(std::ostream& out)
+{
+  out << R"XXX(
+template <class _Fn, class _Sco>
+static inline _CCCL_DEVICE void __cuda_atomic_store_memory_order_dispatch(_Fn &__cuda_store, int __memorder, _Sco) {
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (
+      switch (__memorder) {
+        case __ATOMIC_RELEASE: __cuda_store(__atomic_cuda_release{}); break;
+        case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __atomic_cuda_seq_cst{}); [[fallthrough]];
+        case __ATOMIC_RELAXED: __cuda_store(__atomic_cuda_relaxed{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    ),
+    NV_IS_DEVICE, (
+      switch (__memorder) {
+        case __ATOMIC_RELEASE: [[fallthrough]];
+        case __ATOMIC_SEQ_CST: __cuda_atomic_membar(_Sco{}); [[fallthrough]];
+        case __ATOMIC_RELAXED: __cuda_store(__atomic_cuda_volatile{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    )
+  )
+}
+)XXX";
+  // Argument ID Reference
+  // 0 - Operand Type
+  // 1 - Operand Size
+  // 2 - Constraint
+  // 3 - Memory order
+  // 4 - Memory order semantic
+  // 5 - Scope tag
+  // 6 - Scope semantic
+  // 7 - Mmio tag
+  // 8 - Mmio semantic
+  constexpr auto asm_intrinsic_format_128 = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_store(
+  _Type* __ptr, _Type& __val, {3}, __atomic_cuda_operand_{0}{1}, {5}, {7})
+{{
+  static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840");
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (),
+    NV_ANY_TARGET, (__atomic_ldst_128b_unsupported_before_SM_70();)
+  )
+  asm volatile(R"YYY(
+    {{
+      .reg .b128 _v;
+      mov.b128 _v, {{%1, %2}};
+      st{8}{4}{6}.b128 [%0],_v;
+    }}
+  )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory");
+}})XXX";
+  constexpr auto asm_intrinsic_format     = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_store(
+  _Type* __ptr, _Type& __val, {3}, __atomic_cuda_operand_{0}{1}, {5}, {7})
+{{ asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__val) : "memory"); }})XXX";
+
+  constexpr size_t supported_sizes[] = {
+    16,
+    32,
+    64,
+    128,
+  };
+
+  constexpr Operand supported_types[] = {
+    Operand::Bit,
+  };
+
+  constexpr Semantic supported_semantics[] = {
+    Semantic::Release,
+    Semantic::Relaxed,
+    Semantic::Volatile,
+  };
+
+  constexpr Scope supported_scopes[] = {
+    Scope::CTA,
+    Scope::Cluster,
+    Scope::GPU,
+    Scope::System,
+  };
+
+  constexpr Mmio mmio_states[] = {
+    Mmio::Disabled,
+    Mmio::Enabled,
+  };
+
+  for (auto size : supported_sizes)
+  {
+    for (auto type : supported_types)
+    {
+      for (auto sem : supported_semantics)
+      {
+        for (auto sco : supported_scopes)
+        {
+          for (auto mm : mmio_states)
+          {
+            if (size == 16 && type == Operand::Floating)
+            {
+              continue;
+            }
+            if (size == 128 && type != Operand::Bit)
+            {
+              continue;
+            }
+            if ((mm == Mmio::Enabled) && ((sco != Scope::System) || (sem != Semantic::Relaxed)))
+            {
+              continue;
+            }
+
+            if (size == 128)
+            {
+              out << std::format(
+                asm_intrinsic_format_128,
+                /* 0 */ operand(type),
+                /* 1 */ size,
+                /* 2 */ constraints(type, size),
+                /* 3 */ semantic_tag(sem),
+                /* 4 */ semantic_ld_st(sem),
+                /* 5 */ scope_tag(sco),
+                /* 6 */ scope_ld_st(sem, sco),
+                /* 7 */ mmio_tag(mm),
+                /* 8 */ mmio(mm));
+            }
+            else
+            {
+              out << std::format(
+                asm_intrinsic_format,
+                /* 0 */ operand(type),
+                /* 1 */ size,
+                /* 2 */ constraints(type, size),
+                /* 3 */ semantic_tag(sem),
+                /* 4 */ semantic_ld_st(sem),
+                /* 5 */ scope_tag(sco),
+                /* 6 */ scope_ld_st(sem, sco),
+                /* 7 */ mmio_tag(mm),
+                /* 8 */ mmio(mm));
+            }
+          }
+        }
+      }
+    }
+  }
+  out << "\n"
+      << R"XXX(
+template <typename _Type, typename _Tag, typename _Sco, typename _Mmio>
+struct __cuda_atomic_bind_store {
+  _Type* __ptr;
+  _Type* __val;
+
+  template <typename _Atomic_Memorder>
+  inline _CCCL_DEVICE void operator()(_Atomic_Memorder) {
+    __cuda_atomic_store(__ptr, *__val, _Atomic_Memorder{}, _Tag{}, _Sco{}, _Mmio{});
+  }
+};
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE void __atomic_store_cuda(_Type* __ptr, _Type& __val, int __memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr);
+  __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__val);
+  if (__cuda_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {{return;}}
+  __cuda_atomic_bind_store<__proxy_t, __proxy_tag, _Sco, __atomic_cuda_mmio_disable> __bound_store{__ptr_proxy, __val_proxy};
+  __cuda_atomic_store_memory_order_dispatch(__bound_store, __memorder, _Sco{});
+}
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE void __atomic_store_cuda(volatile _Type* __ptr, _Type& __val, int __memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr));
+  __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__val);
+  if (__cuda_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {{return;}}
+  __cuda_atomic_bind_store<__proxy_t, __proxy_tag, _Sco, __atomic_cuda_mmio_disable> __bound_store{__ptr_proxy, __val_proxy};
+  __cuda_atomic_store_memory_order_dispatch(__bound_store, __memorder, _Sco{});
+}
+)XXX";
+}
+
+#endif // LD_ST_H