[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/libcudacxx/codegen/.gitignore

@@ -0,0 +1 @@
+build

cccl_upstream/libcudacxx/codegen/CMakeLists.txt

@@ -0,0 +1,51 @@
+## Codegen adds the following build targets
+#    libcudacxx.atomics.codegen
+#    libcudacxx.atomics.codegen.install
+## Test targets:
+#    libcudacxx.test.atomics.codegen.diff
+
+add_executable(codegen EXCLUDE_FROM_ALL codegen.cpp)
+
+target_compile_features(codegen PRIVATE cxx_std_20)
+
+set(
+  atomic_generated_output
+  "${libcudacxx_BINARY_DIR}/codegen/cuda_ptx_generated.h"
+)
+set(
+  atomic_install_location
+  "${libcudacxx_SOURCE_DIR}/include/cuda/std/__atomic/functions"
+)
+
+add_custom_target(
+  libcudacxx.atomics.codegen
+  COMMAND codegen "${atomic_generated_output}"
+  BYPRODUCTS "${atomic_generated_output}"
+)
+
+add_custom_target(
+  libcudacxx.atomics.codegen.install
+  # gersemi: off
+  COMMAND
+    "${CMAKE_COMMAND}" -E copy
+      "${atomic_generated_output}"
+      "${atomic_install_location}/cuda_ptx_generated.h"
+  # gersemi: on
+  DEPENDS libcudacxx.atomics.codegen
+  BYPRODUCTS "${atomic_install_location}/cuda_ptx_generated.h"
+)
+
+add_test(
+  NAME libcudacxx.test.atomics.codegen.diff
+  # gersemi: off
+  COMMAND
+    "${CMAKE_COMMAND}" -E compare_files
+      "${atomic_install_location}/cuda_ptx_generated.h"
+      "${atomic_generated_output}"
+  # gersemi: on
+)
+
+set_tests_properties(
+  libcudacxx.test.atomics.codegen.diff
+  PROPERTIES REQUIRED_FILES "${atomic_generated_output}"
+)

cccl_upstream/libcudacxx/codegen/add_ptx_instruction.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+
+##===----------------------------------------------------------------------===##
+##
+## Part of libcu++, the C++ Standard Library for your entire system,
+## under the Apache License v2.0 with LLVM Exceptions.
+## See https://llvm.org/LICENSE.txt for license information.
+## SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+## SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+##
+##===----------------------------------------------------------------------===##
+
+import argparse
+import os
+
+import cccl_paths
+
+docs = os.path.join(cccl_paths.DOCS_LIBCUDACXX_DIR, "ptx", "instructions")
+test = os.path.join(cccl_paths.LIBCUDACXX_TEST_DIR, "libcudacxx", "cuda", "ptx")
+src = os.path.join(cccl_paths.LIBCUDACXX_INCLUDE_DIR, "cuda", "__ptx", "instructions")
+ptx_header = os.path.join(cccl_paths.LIBCUDACXX_INCLUDE_DIR, "cuda", "ptx")
+instr_docs = os.path.join(cccl_paths.DOCS_LIBCUDACXX_DIR, "ptx", "instructions.rst")
+
+
+def add_docs(ptx_instr, url):
+    cpp_instr = ptx_instr.replace(".", "_")
+    underbar = "=" * len(ptx_instr)
+
+    (docs / f"{cpp_instr}.rst").write_text(
+        f""".. _libcudacxx-ptx-instructions-{ptx_instr.replace(".", "-")}:
+
+{ptx_instr}
+{underbar}
+
+-  PTX ISA:
+   `{ptx_instr} <{url}>`__
+
+.. include:: generated/{cpp_instr}.rst
+"""
+    )
+
+
+def add_test(ptx_instr):
+    cpp_instr = ptx_instr.replace(".", "_")
+    dst = test / f"ptx.{ptx_instr}.compile.pass.cpp"
+    dst.write_text(
+        f"""//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+// UNSUPPORTED: libcpp-has-no-threads
+
+// <cuda/ptx>
+
+#include <cuda/ptx>
+#include <cuda/std/utility>
+
+#include "generated/{cpp_instr}.h"
+
+int main(int, char**)
+{{
+  return 0;
+}}
+"""
+    )
+
+
+def add_src(ptx_instr):
+    cpp_instr = ptx_instr.replace(".", "_")
+    (src / f"{cpp_instr}.h").write_text(
+        f"""// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _CUDA_PTX_{cpp_instr.upper()}_H_
+#define _CUDA_PTX_{cpp_instr.upper()}_H_
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/__ptx/ptx_dot_variants.h>
+#include <cuda/__ptx/ptx_helper_functions.h>
+#include <cuda/std/cstdint>
+
+#include <nv/target> // __CUDA_MINIMUM_ARCH__ and friends
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_PTX
+
+#include <cuda/__ptx/instructions/generated/{cpp_instr}.h>
+
+_CCCL_END_NAMESPACE_CUDA_PTX
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_PTX_{cpp_instr.upper()}_H_
+"""
+    )
+
+
+def add_ptx_header_include(ptx_instr):
+    cpp_instr = ptx_instr.replace(".", "_")
+    txt = ptx_header.read_text()
+    # just add as first new include. clang-format will sort it in
+    idx = txt.index("#include <cuda/__ptx/instructions")
+    txt = (
+        txt[:idx]
+        + f"""#include <cuda/__ptx/instructions/{cpp_instr}.h>\n"""
+        + txt[idx:]
+    )
+    ptx_header.write_text(txt)
+
+
+def add_docs_include(ptx_instr):
+    cpp_instr = ptx_instr.replace(".", "_")
+    txt = instr_docs.read_text()
+    # just add as first new include
+    idx = txt.index("   instructions/")
+    txt = txt[:idx] + f"   instructions/{cpp_instr}\n" + txt[idx:]
+    instr_docs.write_text(txt)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("ptx_instruction", type=str)
+    parser.add_argument("url", type=str)
+
+    args = parser.parse_args()
+
+    ptx_instr = args.ptx_instruction
+    url = args.url
+
+    # Enable using internal urls in the command-line, to be automatically converted to public URLs.
+    if url.startswith("index.html"):
+        url = url.replace(
+            "index.html",
+            "https://docs.nvidia.com/cuda/parallel-thread-execution/index.html",
+        )
+
+    add_test(ptx_instr)
+    add_docs(ptx_instr, url)
+    add_src(ptx_instr)
+    add_ptx_header_include(ptx_instr)
+    add_docs_include(ptx_instr)

cccl_upstream/libcudacxx/codegen/cccl_paths.py

@@ -0,0 +1,21 @@
+##===----------------------------------------------------------------------===##
+##
+## Part of libcu++, the C++ Standard Library for your entire system,
+## under the Apache License v2.0 with LLVM Exceptions.
+## See https://llvm.org/LICENSE.txt for license information.
+## SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+## SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+##
+##===----------------------------------------------------------------------===##
+
+import os
+
+LIBCUDACXX_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+LIBCUDACXX_CMAKE_DIR = os.path.join(LIBCUDACXX_DIR, "cmake")
+LIBCUDACXX_CODEGEN_DIR = os.path.join(LIBCUDACXX_DIR, "codegen")
+LIBCUDACXX_INCLUDE_DIR = os.path.join(LIBCUDACXX_DIR, "include")
+LIBCUDACXX_TEST_DIR = os.path.join(LIBCUDACXX_DIR, "test")
+
+DOCS_DIR = os.path.dirname(LIBCUDACXX_DIR)
+DOCS_LIBCUDACXX_DIR = os.path.join(DOCS_DIR, "libcudacxx")

cccl_upstream/libcudacxx/codegen/codegen.cpp

@@ -0,0 +1,45 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <fstream>
+#include <iostream>
+#include <ostream>
+
+#include "generators/compare_and_swap.h"
+#include "generators/exchange.h"
+#include "generators/fence.h"
+#include "generators/fetch_ops.h"
+#include "generators/header.h"
+#include "generators/ld_st.h"
+
+using namespace std::string_literals;
+
+int main(int argc, char** argv)
+{
+  std::fstream filestream;
+
+  if (argc == 2)
+  {
+    filestream.open(argv[1], filestream.out);
+  }
+
+  std::ostream& stream = filestream.is_open() ? filestream : std::cout;
+
+  FormatHeader(stream);
+  FormatFence(stream);
+  FormatLoad(stream);
+  FormatStore(stream);
+  FormatCompareAndSwap(stream);
+  FormatExchange(stream);
+  FormatFetchOps(stream);
+  FormatTail(stream);
+
+  return 0;
+}

cccl_upstream/libcudacxx/codegen/generate_prologue_epilogue.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+
+##===----------------------------------------------------------------------===##
+##
+## Part of libcu++, the C++ Standard Library for your entire system,
+## under the Apache License v2.0 with LLVM Exceptions.
+## See https://llvm.org/LICENSE.txt for license information.
+## SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+## SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+##
+##===----------------------------------------------------------------------===##
+
+import datetime
+import os
+
+import cccl_paths
+
+PROLOGUE_FILE = os.path.join(
+    cccl_paths.LIBCUDACXX_INCLUDE_DIR, "cuda", "std", "__cccl", "prologue.h"
+)
+EPILOGUE_FILE = os.path.join(
+    cccl_paths.LIBCUDACXX_INCLUDE_DIR, "cuda", "std", "__cccl", "epilogue.h"
+)
+
+HEADER = f"""\
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) {datetime.datetime.now().year} NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+// !!! DO NOT EDIT THIS FILE !!! This file is generated by utils/generate_prologue_epilogue.py.
+
+// NO include guards here (this file is included multiple times)"""
+
+FOOTER = """\
+// NO include guards here (this file is included multiple times)
+"""
+
+PUSH_POP_MACROS = {
+    "__declspec modifiers": [
+        "align",
+        "allocate",
+        "allocator",
+        "appdomain",
+        "code_seg",
+        "deprecated",
+        "dllimport",
+        "dllexport",
+        "empty_bases",
+        "hybrid_patchable",
+        "jitintrinsic",
+        "lifetimebound",
+        "naked",
+        "noalias",
+        "noinline",
+        "noreturn",
+        "nothrow",
+        "novtable",
+        "no_sanitize_address",
+        "process",
+        "property",
+        "restrict",
+        "safebuffers",
+        "selectany",
+        "spectre",
+        "thread",
+        "uuid",
+    ],
+    "[[msvc::attribute]] attributes": [
+        "msvc",
+        "flatten",
+        "forceinline",
+        "forceinline_calls",
+        "intrinsic",
+        "noinline",
+        "noinline_calls",
+        "no_tls_guard",
+    ],
+    "Windows nasty macros": ["min", "max", "interface"],
+    "sal.h on Windows": ["__valid", "__callback"],
+    "other macros": ["clang"],
+    "sys/sysmacros.h on linux": ["major", "minor", "makedev"],
+}
+
+
+def write_section(file, section):
+    file.write(section)
+    file.write("\n\n")
+
+
+def make_prologue(file):
+    # Write common header.
+    write_section(file, HEADER)
+
+    # Add prologue/epilogue include logic check.
+    write_section(
+        file,
+        """\
+#if defined(_CCCL_PROLOGUE_INCLUDED)
+#  error \\
+    "cccl internal error: <cuda/std/__cccl/epilogue.h> must be included before next <cuda/std/__cccl/prologue.h> is reincluded"
+#endif
+#define _CCCL_PROLOGUE_INCLUDED() 1""",
+    )
+
+    # Add necessary includes.
+    write_section(
+        file,
+        """\
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/diagnostic.h>
+#include <cuda/std/__cccl/dialect.h>""",
+    )
+
+    # Add push macros.
+    for group_name, macros in PUSH_POP_MACROS.items():
+        write_section(file, f"// {group_name}")
+        for macro in macros:
+            write_section(
+                file,
+                f"""\
+#if defined({macro})
+#  pragma push_macro("{macro}")
+#  undef {macro}
+#  define _CCCL_POP_MACRO_{macro}
+#endif // defined({macro})""",
+            )
+
+    # Add warnings suppressions.
+    write_section(
+        file,
+        '''\
+_CCCL_DIAG_PUSH
+_CCCL_NV_DIAG_PUSH()
+
+// disable some msvc warnings
+// https://github.com/microsoft/STL/blob/master/stl/inc/yvals_core.h#L353
+// warning C4100: 'quack': unreferenced formal parameter
+// warning C4127: conditional expression is constant
+// warning C4180: qualifier applied to function type has no meaning; ignored
+// warning C4197: 'purr': top-level volatile in cast is ignored
+// warning C4324: 'roar': structure was padded due to alignment specifier
+// warning C4455: literal suffix identifiers that do not start with an underscore are reserved
+// warning C4503: 'hum': decorated name length exceeded, name was truncated
+// warning C4522: 'woof' : multiple assignment operators specified
+// warning C4668: 'meow' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif'
+// warning C4800: 'boo': forcing value to bool 'true' or 'false' (performance warning)
+// warning C4996: 'meow': was declared deprecated
+_CCCL_DIAG_SUPPRESS_MSVC(4100 4127 4180 4197 4296 4324 4455 4503 4522 4668 4800 4996)
+
+// Suppress compiler warnings about C++ extensions.
+
+#if _CCCL_COMPILER(GCC, >=, 12)
+_CCCL_DIAG_SUPPRESS_GCC("-Wc++20-extensions")
+_CCCL_DIAG_SUPPRESS_GCC("-Wc++23-extensions")
+#endif // _CCCL_COMPILER(GCC, >=, 12)
+#if _CCCL_COMPILER(GCC, >=, 14)
+_CCCL_DIAG_SUPPRESS_GCC("-Wc++26-extensions")
+#endif // _CCCL_COMPILER(GCC, >=, 14)
+
+_CCCL_DIAG_SUPPRESS_CLANG("-Wc++20-extensions")
+#if _CCCL_COMPILER(CLANG, >=, 17)
+_CCCL_DIAG_SUPPRESS_CLANG("-Wc++23-extensions")
+_CCCL_DIAG_SUPPRESS_CLANG("-Wc++26-extensions")
+#else // ^^^ _CCCL_COMPILER(CLANG, >=, 17) ^^^ / vvv _CCCL_COMPILER(CLANG, <, 17) vvv
+_CCCL_DIAG_SUPPRESS_CLANG("-Wc++2b-extensions")
+#endif // ^^^ _CCCL_COMPILER(CLANG, <, 17) ^^^
+
+// Suppress `if consteval`-related warnings.
+
+_CCCL_DIAG_SUPPRESS_NVHPC(if_consteval_nonstandard)
+_CCCL_DIAG_SUPPRESS_NVHPC(is_constant_evaluated_in_nonconstexpr_context)
+_CCCL_DIAG_SUPPRESS_NVHPC(if_consteval_in_nonconstexpr_function)
+
+_CCCL_DIAG_SUPPRESS_NVCC(3215) // "if consteval" and "if not consteval" are not standard in this mode
+_CCCL_DIAG_SUPPRESS_NVCC(3206) // "if consteval" and "if not consteval" are meaningless in a non-constexpr function
+_CCCL_DIAG_SUPPRESS_NVCC(3060) // call to __builtin_is_constant_evaluated appearing in a non-constexpr function always
+                               // produces "false"''',
+    )
+
+    # Write the common footer.
+    file.write(FOOTER)
+
+
+def make_epilogue(file):
+    # Write common header.
+    write_section(file, HEADER)
+
+    # Write includes.
+    write_section(
+        file,
+        """\
+#include <cuda/std/__cccl/compiler.h>
+#include <cuda/std/__cccl/diagnostic.h>""",
+    )
+
+    # Add prologue/epilogue include logic check.
+    write_section(
+        file,
+        """\
+#if !defined(_CCCL_PROLOGUE_INCLUDED)
+#  error "cccl internal error: <cuda/std/__cccl/prologue.h> must be included before <cuda/std/__cccl/epilogue.h>"
+#endif
+#undef _CCCL_PROLOGUE_INCLUDED""",
+    )
+
+    # Pop warning suppressions.
+    write_section(
+        file,
+        """\
+_CCCL_NV_DIAG_POP()
+_CCCL_DIAG_POP""",
+    )
+
+    # Add pop macros.
+    for group_name, macros in PUSH_POP_MACROS.items():
+        write_section(file, f"// {group_name}")
+        for macro in macros:
+            write_section(
+                file,
+                f"""\
+#if defined({macro})
+#  error \\
+    "cccl internal error: macro `{macro}` was redefined between <cuda/std/__cccl/prologue.h> and <cuda/std/__cccl/epilogue.h>"
+#elif defined(_CCCL_POP_MACRO_{macro})
+#  pragma pop_macro("{macro}")
+#  undef _CCCL_POP_MACRO_{macro}
+#endif""",
+            )
+
+    # Write the common footer.
+    file.write(FOOTER)
+
+
+if __name__ == "__main__":
+    with open(PROLOGUE_FILE, "w") as file:
+        make_prologue(file)
+
+    with open(EPILOGUE_FILE, "w") as file:
+        make_epilogue(file)

cccl_upstream/libcudacxx/codegen/generators/compare_and_swap.h

@@ -0,0 +1,201 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef COMPARED_AND_SWAP_H
+#define COMPARED_AND_SWAP_H
+
+#include <format>
+#include <string>
+
+#include "definitions.h"
+
+inline void FormatCompareAndSwap(std::ostream& out)
+{
+  out << R"XXX(
+template <class _Fn, class _Sco>
+static inline _CCCL_DEVICE bool __cuda_atomic_compare_swap_memory_order_dispatch(_Fn& __cuda_cas, int __success_memorder, int __failure_memorder, _Sco) {
+  bool __res = false;
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (
+      switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) {
+        case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __atomic_cuda_seq_cst{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __res = __cuda_cas(__atomic_cuda_acquire{}); break;
+        case __ATOMIC_ACQ_REL: __res = __cuda_cas(__atomic_cuda_acq_rel{}); break;
+        case __ATOMIC_RELEASE: __res = __cuda_cas(__atomic_cuda_release{}); break;
+        case __ATOMIC_RELAXED: __res = __cuda_cas(__atomic_cuda_relaxed{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    ),
+    NV_IS_DEVICE, (
+      switch (__stronger_order_cuda(__success_memorder, __failure_memorder)) {
+        case __ATOMIC_SEQ_CST: [[fallthrough]];
+        case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __res = __cuda_cas(__atomic_cuda_volatile{}); __cuda_atomic_membar(_Sco{}); break;
+        case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __res = __cuda_cas(__atomic_cuda_volatile{}); break;
+        case __ATOMIC_RELAXED: __res = __cuda_cas(__atomic_cuda_volatile{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    )
+  )
+  return __res;
+}
+)XXX";
+
+  // Argument ID Reference
+  // 0 - Operand Type
+  // 1 - Operand Size
+  // 2 - Type Constraint
+  // 3 - Memory Order
+  // 4 - Memory Order function tag
+  // 5 - Scope Constraint
+  // 6 - Scope function tag
+  constexpr auto asm_intrinsic_format_128 = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange(
+  _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, {4}, __atomic_cuda_operand_{0}{1}, {6})
+{{
+  static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b CAS is not supported until PTX ISA version 840");
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_90, (),
+    NV_ANY_TARGET, (__atomic_cas_128b_unsupported_before_SM_90();)
+  )
+  asm volatile(R"YYY(
+    {{
+      .reg .b128 _d;
+      .reg .b128 _v;
+      mov.b128 _d, {{%3, %4}};
+      mov.b128 _v, {{%5, %6}};
+      atom.cas{3}{5}.b128 _d,[%2],_d,_v;
+      mov.b128 {{%0, %1}}, _d;
+    }}
+  )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr), "l"(__cmp.__x),"l"(__cmp.__y), "l"(__op.__x),"l"(__op.__y) : "memory"); return __dst.__x == __cmp.__x && __dst.__y == __cmp.__y; }})XXX";
+
+  constexpr auto asm_intrinsic_format = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE bool __cuda_atomic_compare_exchange(
+  _Type* __ptr, _Type& __dst, _Type __cmp, _Type __op, {4}, __atomic_cuda_operand_{0}{1}, {6})
+{{ asm volatile("atom.cas{3}{5}.{0}{1} %0,[%1],%2,%3;" : "={2}"(__dst) : "l"(__ptr), "{2}"(__cmp), "{2}"(__op) : "memory"); return __dst == __cmp; }})XXX";
+
+  constexpr Operand supported_types[] = {
+    Operand::Bit,
+  };
+
+  constexpr size_t supported_sizes[] = {
+    32,
+    64,
+    128,
+  };
+
+  constexpr Semantic supported_semantics[] = {
+    Semantic::Acquire,
+    Semantic::Relaxed,
+    Semantic::Release,
+    Semantic::Acq_Rel,
+    Semantic::Volatile,
+  };
+
+  constexpr Scope supported_scopes[] = {
+    Scope::CTA,
+    Scope::Cluster,
+    Scope::GPU,
+    Scope::System,
+  };
+
+  for (auto size : supported_sizes)
+  {
+    for (auto type : supported_types)
+    {
+      for (auto sem : supported_semantics)
+      {
+        for (auto sco : supported_scopes)
+        {
+          if (size == 2 && type != Operand::Bit)
+          {
+            continue;
+          }
+          if (size == 128 && type != Operand::Bit)
+          {
+            continue;
+          }
+
+          if (size == 128)
+          {
+            out << std::format(
+              asm_intrinsic_format_128,
+              operand(type),
+              size,
+              constraints(type, size),
+              semantic(sem),
+              semantic_tag(sem),
+              scope(sco),
+              scope_tag(sco));
+          }
+          else
+          {
+            out << std::format(
+              asm_intrinsic_format,
+              operand(type),
+              size,
+              constraints(type, size),
+              semantic(sem),
+              semantic_tag(sem),
+              scope(sco),
+              scope_tag(sco));
+          }
+        }
+      }
+    }
+  }
+
+  out << "\n"
+      << R"XXX(
+template <typename _Type, typename _Tag, typename _Sco>
+struct __cuda_atomic_bind_compare_exchange {
+  _Type* __ptr;
+  _Type* __exp;
+  _Type* __des;
+
+  template <typename _Atomic_Memorder>
+  inline _CCCL_DEVICE bool operator()(_Atomic_Memorder) {
+    return __cuda_atomic_compare_exchange(__ptr, *__exp, *__exp, *__des, _Atomic_Memorder{}, _Tag{}, _Sco{});
+  }
+};
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE bool __atomic_compare_exchange_cuda(_Type* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr);
+  __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp);
+  __proxy_t* __des_proxy  = reinterpret_cast<__proxy_t*>(&__des);
+  bool __res = false;
+  if (__cuda_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;}
+  __cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy};
+  return __cuda_atomic_compare_swap_memory_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{});
+}
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE bool __atomic_compare_exchange_cuda(_Type volatile* __ptr, _Type* __exp, _Type __des, bool, int __success_memorder, int __failure_memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr));
+  __proxy_t* __exp_proxy = reinterpret_cast<__proxy_t*>(__exp);
+  __proxy_t* __des_proxy  = reinterpret_cast<__proxy_t*>(&__des);
+  bool __res = false;
+  if (__cuda_compare_exchange_weak_if_local(__ptr_proxy, __exp_proxy, __des_proxy, &__res)) {return __res;}
+  __cuda_atomic_bind_compare_exchange<__proxy_t, __proxy_tag, _Sco> __bound_compare_swap{__ptr_proxy, __exp_proxy, __des_proxy};
+  return __cuda_atomic_compare_swap_memory_order_dispatch(__bound_compare_swap, __success_memorder, __failure_memorder, _Sco{});
+}
+)XXX";
+}
+
+#endif // COMPARED_AND_SWAP_H

cccl_upstream/libcudacxx/codegen/generators/definitions.h

@@ -0,0 +1,192 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef DEFINITIONS_H
+#define DEFINITIONS_H
+
+#include <format>
+#include <map>
+#include <string>
+#include <type_traits>
+#include <vector>
+
+enum class Mmio
+{
+  Disabled,
+  Enabled,
+};
+
+inline std::string mmio(Mmio m)
+{
+  static const char* mmio_map[]{
+    "",
+    ".mmio",
+  };
+  return mmio_map[std::underlying_type_t<Mmio>(m)];
+}
+
+inline std::string mmio_tag(Mmio m)
+{
+  static const char* mmio_map[]{
+    "__atomic_cuda_mmio_disable",
+    "__atomic_cuda_mmio_enable",
+  };
+  return mmio_map[std::underlying_type_t<Mmio>(m)];
+}
+
+enum class Operand
+{
+  Floating,
+  Unsigned,
+  Signed,
+  Bit,
+};
+
+inline std::string operand(Operand op)
+{
+  static std::map op_map = {
+    std::pair{Operand::Floating, "f"},
+    std::pair{Operand::Unsigned, "u"},
+    std::pair{Operand::Signed, "s"},
+    std::pair{Operand::Bit, "b"},
+  };
+  return op_map[op];
+}
+
+inline std::string operand_proxy_type(Operand op, size_t sz)
+{
+  if (op == Operand::Floating)
+  {
+    if (sz == 32)
+    {
+      return {"float"};
+    }
+    else
+    {
+      return {"double"};
+    }
+  }
+  else if (op == Operand::Signed)
+  {
+    return std::format("int{}_t", sz);
+  }
+  // Binary and unsigned can be the same proxy_type
+  return std::format("uint{}_t", sz);
+}
+
+inline std::string constraints(Operand op, size_t sz)
+{
+  static std::map constraint_map = {
+    std::pair{32,
+              std::map{
+                std::pair{Operand::Bit, "r"},
+                std::pair{Operand::Unsigned, "r"},
+                std::pair{Operand::Signed, "r"},
+                std::pair{Operand::Floating, "f"},
+              }},
+    std::pair{64,
+              std::map{
+                std::pair{Operand::Bit, "l"},
+                std::pair{Operand::Unsigned, "l"},
+                std::pair{Operand::Signed, "l"},
+                std::pair{Operand::Floating, "d"},
+              }},
+    std::pair{128,
+              std::map{
+                std::pair{Operand::Bit, "l"},
+                std::pair{Operand::Unsigned, "l"},
+                std::pair{Operand::Signed, "l"},
+                std::pair{Operand::Floating, "d"},
+              }},
+  };
+
+  if (sz == 16)
+  {
+    return {"h"};
+  }
+  else
+  {
+    return constraint_map[sz][op];
+  }
+}
+
+enum class Semantic
+{
+  Relaxed,
+  Release,
+  Acquire,
+  Acq_Rel,
+  Seq_Cst,
+  Volatile,
+};
+
+inline std::string semantic(Semantic sem)
+{
+  static std::map sem_map = {
+    std::pair{Semantic::Relaxed, ".relaxed"},
+    std::pair{Semantic::Release, ".release"},
+    std::pair{Semantic::Acquire, ".acquire"},
+    std::pair{Semantic::Acq_Rel, ".acq_rel"},
+    std::pair{Semantic::Seq_Cst, ".sc"},
+    std::pair{Semantic::Volatile, ""},
+  };
+  return sem_map[sem];
+}
+
+inline std::string semantic_tag(Semantic sem)
+{
+  static std::map sem_map = {
+    std::pair{Semantic::Relaxed, "__atomic_cuda_relaxed"},
+    std::pair{Semantic::Release, "__atomic_cuda_release"},
+    std::pair{Semantic::Acquire, "__atomic_cuda_acquire"},
+    std::pair{Semantic::Acq_Rel, "__atomic_cuda_acq_rel"},
+    std::pair{Semantic::Seq_Cst, "__atomic_cuda_seq_cst"},
+    std::pair{Semantic::Volatile, "__atomic_cuda_volatile"},
+  };
+  return sem_map[sem];
+}
+
+enum class Scope
+{
+  Thread,
+  Warp,
+  CTA,
+  Cluster,
+  GPU,
+  System,
+};
+
+inline std::string scope(Scope sco)
+{
+  static std::map sco_map = {
+    std::pair{Scope::Thread, ""},
+    std::pair{Scope::Warp, ""},
+    std::pair{Scope::CTA, ".cta"},
+    std::pair{Scope::Cluster, ".cluster"},
+    std::pair{Scope::GPU, ".gpu"},
+    std::pair{Scope::System, ".sys"},
+  };
+  return sco_map[sco];
+}
+
+inline std::string scope_tag(Scope sco)
+{
+  static std::map sco_map = {
+    std::pair{Scope::Thread, "__thread_scope_thread_tag"},
+    std::pair{Scope::Warp, ""},
+    std::pair{Scope::CTA, "__thread_scope_block_tag"},
+    std::pair{Scope::Cluster, "__thread_scope_cluster_tag"},
+    std::pair{Scope::GPU, "__thread_scope_device_tag"},
+    std::pair{Scope::System, "__thread_scope_system_tag"},
+  };
+  return sco_map[sco];
+}
+
+#endif // DEFINITIONS_H

cccl_upstream/libcudacxx/codegen/generators/exchange.h

@@ -0,0 +1,197 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef EXCHANGE_H
+#define EXCHANGE_H
+
+#include <format>
+#include <string>
+
+#include "definitions.h"
+
+inline void FormatExchange(std::ostream& out)
+{
+  out << R"XXX(
+template <class _Fn, class _Sco>
+static inline _CCCL_DEVICE void __cuda_atomic_exchange_memory_order_dispatch(_Fn& __cuda_exch, int __memorder, _Sco) {
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __atomic_cuda_seq_cst{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __cuda_exch(__atomic_cuda_acquire{}); break;
+        case __ATOMIC_ACQ_REL: __cuda_exch(__atomic_cuda_acq_rel{}); break;
+        case __ATOMIC_RELEASE: __cuda_exch(__atomic_cuda_release{}); break;
+        case __ATOMIC_RELAXED: __cuda_exch(__atomic_cuda_relaxed{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    ),
+    NV_IS_DEVICE, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: [[fallthrough]];
+        case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __cuda_exch(__atomic_cuda_volatile{}); __cuda_atomic_membar(_Sco{}); break;
+        case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __cuda_exch(__atomic_cuda_volatile{}); break;
+        case __ATOMIC_RELAXED: __cuda_exch(__atomic_cuda_volatile{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    )
+  )
+}
+)XXX";
+
+  // Argument ID Reference
+  // 0 - Operand Type
+  // 1 - Operand Size
+  // 2 - Type Constraint
+  // 3 - Memory Order
+  // 4 - Memory Order function tag
+  // 5 - Scope Constraint
+  // 6 - Scope function tag
+  constexpr auto asm_intrinsic_format_128 = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_exchange(
+  _Type* __ptr, _Type& __old, _Type __new, {4}, __atomic_cuda_operand_{0}{1}, {6})
+{{
+  static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b exchange is not supported until PTX ISA version 840");
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_90, (),
+    NV_ANY_TARGET, (__atomic_exchange_128b_unsupported_before_SM_90();)
+  )
+  asm volatile(R"YYY(
+    {{
+      .reg .b128 _d;
+      .reg .b128 _v;
+      mov.b128 _v, {{%3, %4}};
+      atom.exch{3}{5}.b128 _d,[%2],_v;
+      mov.b128 {{%0, %1}}, _d;
+    }}
+  )YYY" : "=l"(__old.__x),"=l"(__old.__y) : "l"(__ptr), "l"(__new.__x),"l"(__new.__y) : "memory");
+}})XXX";
+
+  constexpr auto asm_intrinsic_format = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_exchange(
+  _Type* __ptr, _Type& __old, _Type __new, {4}, __atomic_cuda_operand_{0}{1}, {6})
+{{ asm volatile("atom.exch{3}{5}.{0}{1} %0,[%1],%2;" : "={2}"(__old) : "l"(__ptr), "{2}"(__new) : "memory"); }})XXX";
+
+  constexpr Operand supported_types[] = {
+    Operand::Bit,
+  };
+
+  constexpr size_t supported_sizes[] = {
+    32,
+    64,
+    128,
+  };
+
+  constexpr Semantic supported_semantics[] = {
+    Semantic::Acquire,
+    Semantic::Relaxed,
+    Semantic::Release,
+    Semantic::Acq_Rel,
+    Semantic::Volatile,
+  };
+
+  constexpr Scope supported_scopes[] = {
+    Scope::CTA,
+    Scope::Cluster,
+    Scope::GPU,
+    Scope::System,
+  };
+
+  for (auto size : supported_sizes)
+  {
+    for (auto type : supported_types)
+    {
+      for (auto sem : supported_semantics)
+      {
+        for (auto sco : supported_scopes)
+        {
+          if (size == 2 && type != Operand::Bit)
+          {
+            continue;
+          }
+          if (size == 128 && type != Operand::Bit)
+          {
+            continue;
+          }
+
+          if (size == 128)
+          {
+            out << std::format(
+              asm_intrinsic_format_128,
+              operand(type),
+              size,
+              constraints(type, size),
+              semantic(sem),
+              semantic_tag(sem),
+              scope(sco),
+              scope_tag(sco));
+          }
+          else
+          {
+            out << std::format(
+              asm_intrinsic_format,
+              operand(type),
+              size,
+              constraints(type, size),
+              semantic(sem),
+              semantic_tag(sem),
+              scope(sco),
+              scope_tag(sco));
+          }
+        }
+      }
+    }
+  }
+
+  out << "\n"
+      << R"XXX(
+template <typename _Type, typename _Tag, typename _Sco>
+struct __cuda_atomic_bind_exchange {
+  _Type* __ptr;
+  _Type* __old;
+  _Type* __new;
+
+  template <typename _Atomic_Memorder>
+  inline _CCCL_DEVICE void operator()(_Atomic_Memorder) {
+    __cuda_atomic_exchange(__ptr, *__old, *__new, _Atomic_Memorder{}, _Tag{}, _Sco{});
+  }
+};
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE void __atomic_exchange_cuda(_Type* __ptr, _Type& __old, _Type __new, int __memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr);
+  __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old);
+  __proxy_t* __new_proxy  = reinterpret_cast<__proxy_t*>(&__new);
+  if(__cuda_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}}
+  __cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy};
+  __cuda_atomic_exchange_memory_order_dispatch(__bound_swap, __memorder, _Sco{});
+}
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE void __atomic_exchange_cuda(_Type volatile* __ptr, _Type& __old, _Type __new, int __memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr));
+  __proxy_t* __old_proxy = reinterpret_cast<__proxy_t*>(&__old);
+  __proxy_t* __new_proxy  = reinterpret_cast<__proxy_t*>(&__new);
+  if(__cuda_exchange_weak_if_local(__ptr_proxy, __new_proxy, __old_proxy)) {{return;}}
+  __cuda_atomic_bind_exchange<__proxy_t, __proxy_tag, _Sco> __bound_swap{__ptr_proxy, __old_proxy, __new_proxy};
+  __cuda_atomic_exchange_memory_order_dispatch(__bound_swap, __memorder, _Sco{});
+}
+)XXX";
+}
+
+#endif // EXCHANGE_H

cccl_upstream/libcudacxx/codegen/generators/fence.h

@@ -0,0 +1,110 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FENCE_H
+#define FENCE_H
+
+#include <format>
+#include <string>
+
+#include "definitions.h"
+
+inline std::string membar_scope(Scope sco)
+{
+  static std::map scope_map{
+    std::pair{Scope::GPU, ".gl"},
+    std::pair{Scope::System, ".sys"},
+    std::pair{Scope::CTA, ".cta"},
+  };
+
+  return scope_map[sco];
+}
+
+inline void FormatFence(std::ostream& out)
+{
+  // Argument ID Reference
+  // 0 - Membar scope tag
+  // 1 - Membar scope
+  constexpr auto intrinsic_membar = R"XXX(
+static inline _CCCL_DEVICE void __cuda_atomic_membar({0})
+{{ asm volatile("membar{1};" ::: "memory"); }})XXX";
+
+  const std::map membar_scopes{
+    std::pair{Scope::GPU, ".gl"},
+    std::pair{Scope::System, ".sys"},
+    std::pair{Scope::CTA, ".cta"},
+  };
+
+  for (const auto& sco : membar_scopes)
+  {
+    out << std::format(intrinsic_membar, scope_tag(sco.first), sco.second);
+  }
+
+  // Argument ID Reference
+  // 0 - Fence scope tag
+  // 1 - Fence scope
+  // 2 - Fence order tag
+  // 3 - Fence order
+  constexpr auto intrinsic_fence = R"XXX(
+static inline _CCCL_DEVICE void __cuda_atomic_fence({0}, {2})
+{{ asm volatile("fence{1}{3};" ::: "memory"); }})XXX";
+
+  const Scope fence_scopes[] = {
+    Scope::CTA,
+    Scope::Cluster,
+    Scope::GPU,
+    Scope::System,
+  };
+
+  const Semantic fence_semantics[] = {
+    Semantic::Acq_Rel,
+    Semantic::Seq_Cst,
+  };
+
+  for (const auto& sco : fence_scopes)
+  {
+    for (const auto& sem : fence_semantics)
+    {
+      out << std::format(intrinsic_fence, scope_tag(sco), semantic(sem), semantic_tag(sem), scope(sco));
+    }
+  }
+  out << "\n"
+      << R"XXX(
+template <typename _Sco>
+static inline _CCCL_DEVICE void __atomic_thread_fence_cuda(int __memorder, _Sco) {
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __atomic_cuda_seq_cst{}); break;
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: [[fallthrough]];
+        case __ATOMIC_ACQ_REL: [[fallthrough]];
+        case __ATOMIC_RELEASE: __cuda_atomic_fence(_Sco{}, __atomic_cuda_acq_rel{}); break;
+        case __ATOMIC_RELAXED: break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    ),
+    NV_IS_DEVICE, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: [[fallthrough]];
+        case __ATOMIC_ACQ_REL: [[fallthrough]];
+        case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); break;
+        case __ATOMIC_RELAXED: break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    )
+  )
+}
+)XXX";
+}
+
+#endif // FENCE_H

cccl_upstream/libcudacxx/codegen/generators/fetch_ops.h

@@ -0,0 +1,219 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef FETCH_OPS_H
+#define FETCH_OPS_H
+
+#include <array>
+#include <format>
+#include <string>
+
+#include "definitions.h"
+
+inline std::string fetch_op_skip_v(std::string fetch_op)
+{
+  if (fetch_op == "add")
+  {
+    return "constexpr auto __skip_v = __atomic_ptr_skip_t<_Type>::__skip;";
+  }
+  return "constexpr auto __skip_v = 1;";
+}
+
+inline void FormatFetchOps(std::ostream& out)
+{
+  const std::vector arithmetic_types = {
+    Operand::Floating,
+    Operand::Unsigned,
+    Operand::Signed,
+  };
+
+  const std::vector minmax_types = {
+    Operand::Unsigned,
+    Operand::Signed,
+  };
+
+  const std::vector bitwise_types = {Operand::Bit};
+
+  const std::map op_support_map{
+    std::pair{std::string{"add"}, std::pair{arithmetic_types, std::string{"arithmetic"}}},
+    std::pair{std::string{"min"}, std::pair{minmax_types, std::string{"minmax"}}},
+    std::pair{std::string{"max"}, std::pair{minmax_types, std::string{"minmax"}}},
+    std::pair{std::string{"or"}, std::pair{bitwise_types, std::string{"bitwise"}}},
+    std::pair{std::string{"xor"}, std::pair{bitwise_types, std::string{"bitwise"}}},
+    std::pair{std::string{"and"}, std::pair{bitwise_types, std::string{"bitwise"}}},
+  };
+
+  // Memory order dispatcher
+  out << R"XXX(
+template <class _Fn, class _Sco>
+static inline _CCCL_DEVICE void __cuda_atomic_fetch_memory_order_dispatch(_Fn& __cuda_fetch, int __memorder, _Sco) {
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __atomic_cuda_seq_cst{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __cuda_fetch(__atomic_cuda_acquire{}); break;
+        case __ATOMIC_ACQ_REL: __cuda_fetch(__atomic_cuda_acq_rel{}); break;
+        case __ATOMIC_RELEASE: __cuda_fetch(__atomic_cuda_release{}); break;
+        case __ATOMIC_RELAXED: __cuda_fetch(__atomic_cuda_relaxed{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    ),
+    NV_IS_DEVICE, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: [[fallthrough]];
+        case __ATOMIC_ACQ_REL: __cuda_atomic_membar(_Sco{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __cuda_fetch(__atomic_cuda_volatile{}); __cuda_atomic_membar(_Sco{}); break;
+        case __ATOMIC_RELEASE: __cuda_atomic_membar(_Sco{}); __cuda_fetch(__atomic_cuda_volatile{}); break;
+        case __ATOMIC_RELAXED: __cuda_fetch(__atomic_cuda_volatile{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    )
+  )
+}
+)XXX";
+
+  // Argument ID Reference
+  // 0 - Atomic Operation
+  // 1 - Operand Type
+  // 2 - Operand Size
+  // 3 - Type Constraint
+  // 4 - Memory Order
+  // 5 - Memory Order function tag
+  // 6 - Scope Constraint
+  // 7 - Scope function tag
+  constexpr auto asm_intrinsic_format = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_fetch_{0}(
+  _Type* __ptr, _Type& __dst, _Type __op, {5}, __atomic_cuda_operand_{1}{2}, {7})
+{{ asm volatile("atom.{0}{4}{6}.{1}{2} %0,[%1],%2;" : "={3}"(__dst) : "l"(__ptr), "{3}"(__op) : "memory"); }})XXX";
+
+  // 0 - Atomic Operation
+  // 1 - Operand type constraint
+  // 2 - Pointer op skip_v
+  constexpr auto fetch_bind_invoke = R"XXX(
+template <typename _Type, typename _Tag, typename _Sco>
+struct __cuda_atomic_bind_fetch_{0} {{
+  _Type* __ptr;
+  _Type* __dst;
+  _Type* __op;
+
+  template <typename _Atomic_Memorder>
+  inline _CCCL_DEVICE void operator()(_Atomic_Memorder) {{
+    __cuda_atomic_fetch_{0}(__ptr, *__dst, *__op, _Atomic_Memorder{{}}, _Tag{{}}, _Sco{{}});
+  }}
+}};
+template <class _Type, class _Up, class _Sco, __atomic_enable_if_native_{1}<_Type> = 0>
+[[nodiscard]] static inline _CCCL_DEVICE _Type __atomic_fetch_{0}_cuda(_Type* __ptr, _Up __op, int __memorder, _Sco)
+{{
+  {2}
+  __op = __op * __skip_v;
+  using __proxy_t        = typename __atomic_cuda_deduce_{1}<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_{1}<_Type>::__tag;
+  _Type __dst{{}};
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr);
+  __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst);
+  __proxy_t* __op_proxy  = reinterpret_cast<__proxy_t*>(&__op);
+  if (__cuda_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}}
+  __cuda_atomic_bind_fetch_{0}<__proxy_t, __proxy_tag, _Sco> __bound_{0}{{__ptr_proxy, __dst_proxy, __op_proxy}};
+  __cuda_atomic_fetch_memory_order_dispatch(__bound_{0}, __memorder, _Sco{{}});
+  return __dst;
+}}
+template <class _Type, class _Up, class _Sco, __atomic_enable_if_native_{1}<_Type> = 0>
+[[nodiscard]] static inline _CCCL_DEVICE _Type __atomic_fetch_{0}_cuda(_Type volatile* __ptr, _Up __op, int __memorder, _Sco)
+{{
+  {2}
+  __op = __op * __skip_v;
+  using __proxy_t        = typename __atomic_cuda_deduce_{1}<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_{1}<_Type>::__tag;
+  _Type __dst{{}};
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr));
+  __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst);
+  __proxy_t* __op_proxy  = reinterpret_cast<__proxy_t*>(&__op);
+  if (__cuda_fetch_{0}_weak_if_local(__ptr_proxy, *__op_proxy, __dst_proxy)) {{return __dst;}}
+  __cuda_atomic_bind_fetch_{0}<__proxy_t, __proxy_tag, _Sco> __bound_{0}{{__ptr_proxy, __dst_proxy, __op_proxy}};
+  __cuda_atomic_fetch_memory_order_dispatch(__bound_{0}, __memorder, _Sco{{}});
+  return __dst;
+}}
+)XXX";
+
+  constexpr size_t supported_sizes[] = {
+    32,
+    64,
+  };
+
+  constexpr Semantic supported_semantics[] = {
+    Semantic::Acquire,
+    Semantic::Relaxed,
+    Semantic::Release,
+    Semantic::Acq_Rel,
+    Semantic::Volatile,
+  };
+
+  constexpr Scope supported_scopes[] = {
+    Scope::CTA,
+    Scope::Cluster,
+    Scope::GPU,
+    Scope::System,
+  };
+
+  for (auto& op_kp : op_support_map)
+  {
+    const auto& op_name    = op_kp.first;
+    const auto& op_type_kp = op_kp.second;
+    const auto& type_list  = op_type_kp.first;
+    const auto& deduction  = op_type_kp.second;
+    for (auto type : type_list)
+    {
+      for (auto size : supported_sizes)
+      {
+        const std::string proxy_type = operand_proxy_type(type, size);
+        for (auto sco : supported_scopes)
+        {
+          for (auto sem : supported_semantics)
+          {
+            // There is no atom.add.s64
+            if (op_name == "add" && type == Operand::Signed && size == 64)
+            {
+              continue;
+            }
+            out << std::format(
+              asm_intrinsic_format,
+              /* 0 */ op_name,
+              /* 1 */ operand(type),
+              /* 2 */ size,
+              /* 3 */ constraints(type, size),
+              /* 4 */ semantic(sem),
+              /* 5 */ semantic_tag(sem),
+              /* 6 */ scope(sco),
+              /* 7 */ scope_tag(sco));
+          }
+        }
+      }
+    }
+    out << "\n" << std::format(fetch_bind_invoke, op_name, deduction, fetch_op_skip_v(op_name));
+  }
+
+  out << R"XXX(
+template <class _Type, class _Up, class _Sco>
+[[nodiscard]] static inline _CCCL_DEVICE _Type __atomic_fetch_sub_cuda(_Type* __ptr, _Up __op, int __memorder, _Sco)
+{
+  return __atomic_fetch_add_cuda(__ptr, -__op, __memorder, _Sco{});
+}
+template <class _Type, class _Up, class _Sco>
+[[nodiscard]] static inline _CCCL_DEVICE _Type __atomic_fetch_sub_cuda(_Type volatile* __ptr, _Up __op, int __memorder, _Sco)
+{
+  return __atomic_fetch_add_cuda(__ptr, -__op, __memorder, _Sco{});
+}
+)XXX";
+}
+
+#endif // FETCH_OPS_H

cccl_upstream/libcudacxx/codegen/generators/header.h

@@ -0,0 +1,88 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HEADER_H
+#define HEADER_H
+
+#include <string>
+
+inline void FormatHeader(std::ostream& out)
+{
+  constexpr auto header = R"XXX(//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+// This is an autogenerated file, we want to ensure that it contains exactly the contents we want to generate
+// clang-format off
+
+#ifndef _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_H
+#define _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_H
+
+#include <cuda/std/detail/__config>
+
+#if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC)
+#  pragma GCC system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG)
+#  pragma clang system_header
+#elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC)
+#  pragma system_header
+#endif // no system header
+
+#include <cuda/std/cassert>
+#include <cuda/std/cstdint>
+
+#include <cuda/std/__type_traits/enable_if.h>
+#include <cuda/std/__type_traits/is_signed.h>
+#include <cuda/std/__type_traits/is_unsigned.h>
+
+#include <cuda/std/__atomic/scopes.h>
+#include <cuda/std/__atomic/order.h>
+#include <cuda/std/__atomic/functions/common.h>
+#include <cuda/std/__atomic/functions/cuda_ptx_generated_helper.h>
+#include <cuda/std/__atomic/functions/cuda_local.h>
+
+#include <cuda/std/__cccl/prologue.h>
+
+_CCCL_BEGIN_NAMESPACE_CUDA_STD
+
+#if _CCCL_CUDA_COMPILATION()
+
+extern "C" _CCCL_DEVICE void __atomic_cas_128b_unsupported_before_SM_90();
+extern "C" _CCCL_DEVICE void __atomic_exchange_128b_unsupported_before_SM_90();
+extern "C" _CCCL_DEVICE void __atomic_ldst_128b_unsupported_before_SM_70();
+)XXX";
+
+  out << header;
+}
+
+inline void FormatTail(std::ostream& out)
+{
+  constexpr auto tail = R"XXX(
+#endif // _CCCL_CUDA_COMPILATION()
+
+_CCCL_END_NAMESPACE_CUDA_STD
+
+#include <cuda/std/__cccl/epilogue.h>
+
+#endif // _CUDA_STD___ATOMIC_FUNCTIONS_CUDA_PTX_GENERATED_H
+
+// clang-format on
+)XXX";
+
+  out << tail;
+}
+
+#endif // HEADER_H

cccl_upstream/libcudacxx/codegen/generators/ld_st.h

@@ -0,0 +1,407 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LD_ST_H
+#define LD_ST_H
+
+#include <format>
+#include <string>
+
+#include "definitions.h"
+
+inline std::string semantic_ld_st(Semantic sem)
+{
+  static std::map sem_map = {
+    std::pair{Semantic::Relaxed, ".relaxed"},
+    std::pair{Semantic::Release, ".release"},
+    std::pair{Semantic::Acquire, ".acquire"},
+    std::pair{Semantic::Volatile, ".volatile"},
+  };
+  return sem_map[sem];
+}
+
+inline std::string scope_ld_st(Semantic sem, Scope sco)
+{
+  if (sem == Semantic::Volatile)
+  {
+    return "";
+  }
+  return scope(sco);
+}
+
+inline void FormatLoad(std::ostream& out)
+{
+  out << R"XXX(
+template <class _Fn, class _Sco>
+static inline _CCCL_DEVICE void __cuda_atomic_load_memory_order_dispatch(_Fn &__cuda_load, int __memorder, _Sco) {
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __atomic_cuda_seq_cst{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __cuda_load(__atomic_cuda_acquire{}); break;
+        case __ATOMIC_RELAXED: __cuda_load(__atomic_cuda_relaxed{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    ),
+    NV_IS_DEVICE, (
+      switch (__memorder) {
+        case __ATOMIC_SEQ_CST: __cuda_atomic_membar(_Sco{}); [[fallthrough]];
+        case __ATOMIC_CONSUME: [[fallthrough]];
+        case __ATOMIC_ACQUIRE: __cuda_load(__atomic_cuda_volatile{}); __cuda_atomic_membar(_Sco{}); break;
+        case __ATOMIC_RELAXED: __cuda_load(__atomic_cuda_volatile{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    )
+  )
+}
+)XXX";
+
+  // Argument ID Reference
+  // 0 - Operand Type
+  // 1 - Operand Size
+  // 2 - Constraint
+  // 3 - Memory order
+  // 4 - Memory order semantic
+  // 5 - Scope tag
+  // 6 - Scope semantic
+  // 7 - Mmio tag
+  // 8 - Mmio semantic
+  constexpr auto asm_intrinsic_format_128 = R"XXX(
+  template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_load(
+  const _Type* __ptr, _Type& __dst, {3}, __atomic_cuda_operand_{0}{1}, {5}, {7})
+{{
+  static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840");
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (),
+    NV_ANY_TARGET, (__atomic_ldst_128b_unsupported_before_SM_70();)
+  )
+  asm volatile(R"YYY(
+    {{
+      .reg .b128 _d;
+      ld{8}{4}{6}.b128 _d,[%2];
+      mov.b128 {{%0, %1}}, _d;
+    }}
+  )YYY" : "=l"(__dst.__x),"=l"(__dst.__y) : "l"(__ptr) : "memory");
+}})XXX";
+  constexpr auto asm_intrinsic_format     = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_load(
+  const _Type* __ptr, _Type& __dst, {3}, __atomic_cuda_operand_{0}{1}, {5}, {7})
+{{ asm volatile("ld{8}{4}{6}.{0}{1} %0,[%1];" : "={2}"(__dst) : "l"(__ptr) : "memory"); }})XXX";
+
+  constexpr size_t supported_sizes[] = {
+    16,
+    32,
+    64,
+    128,
+  };
+
+  constexpr Operand supported_types[] = {
+    Operand::Bit,
+    Operand::Floating,
+    Operand::Unsigned,
+    Operand::Signed,
+  };
+
+  constexpr Semantic supported_semantics[] = {
+    Semantic::Acquire,
+    Semantic::Relaxed,
+    Semantic::Volatile,
+  };
+
+  constexpr Scope supported_scopes[] = {
+    Scope::CTA,
+    Scope::Cluster,
+    Scope::GPU,
+    Scope::System,
+  };
+
+  constexpr Mmio mmio_states[] = {
+    Mmio::Disabled,
+    Mmio::Enabled,
+  };
+
+  for (auto size : supported_sizes)
+  {
+    for (auto type : supported_types)
+    {
+      for (auto sem : supported_semantics)
+      {
+        for (auto sco : supported_scopes)
+        {
+          for (auto mm : mmio_states)
+          {
+            if (size == 16 && type == Operand::Floating)
+            {
+              continue;
+            }
+            if (size == 128 && type != Operand::Bit)
+            {
+              continue;
+            }
+            if ((mm == Mmio::Enabled) && ((sco != Scope::System) || (sem != Semantic::Relaxed)))
+            {
+              continue;
+            }
+
+            if (size == 128)
+            {
+              out << std::format(
+                asm_intrinsic_format_128,
+                /* 0 */ operand(type),
+                /* 1 */ size,
+                /* 2 */ constraints(type, size),
+                /* 3 */ semantic_tag(sem),
+                /* 4 */ semantic_ld_st(sem),
+                /* 5 */ scope_tag(sco),
+                /* 6 */ scope_ld_st(sem, sco),
+                /* 7 */ mmio_tag(mm),
+                /* 8 */ mmio(mm));
+            }
+            else
+            {
+              out << std::format(
+                asm_intrinsic_format,
+                /* 0 */ operand(type),
+                /* 1 */ size,
+                /* 2 */ constraints(type, size),
+                /* 3 */ semantic_tag(sem),
+                /* 4 */ semantic_ld_st(sem),
+                /* 5 */ scope_tag(sco),
+                /* 6 */ scope_ld_st(sem, sco),
+                /* 7 */ mmio_tag(mm),
+                /* 8 */ mmio(mm));
+            }
+          }
+        }
+      }
+    }
+  }
+  out << "\n"
+      << R"XXX(
+template <typename _Type, typename _Tag, typename _Sco, typename _Mmio>
+struct __cuda_atomic_bind_load {
+  const _Type* __ptr;
+  _Type* __dst;
+
+  template <typename _Atomic_Memorder>
+  inline _CCCL_DEVICE void operator()(_Atomic_Memorder) {
+    __cuda_atomic_load(__ptr, *__dst, _Atomic_Memorder{}, _Tag{}, _Sco{}, _Mmio{});
+  }
+};
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE void __atomic_load_cuda(const _Type* __ptr, _Type& __dst, int __memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  const __proxy_t* __ptr_proxy = reinterpret_cast<const __proxy_t*>(__ptr);
+  __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst);
+  if (__cuda_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {{return;}}
+  __cuda_atomic_bind_load<__proxy_t, __proxy_tag, _Sco, __atomic_cuda_mmio_disable> __bound_load{__ptr_proxy, __dst_proxy};
+  __cuda_atomic_load_memory_order_dispatch(__bound_load, __memorder, _Sco{});
+}
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE void __atomic_load_cuda(const _Type volatile* __ptr, _Type& __dst, int __memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  const __proxy_t* __ptr_proxy = reinterpret_cast<const __proxy_t*>(const_cast<_Type*>(__ptr));
+  __proxy_t* __dst_proxy = reinterpret_cast<__proxy_t*>(&__dst);
+  if (__cuda_load_weak_if_local(__ptr_proxy, __dst_proxy, sizeof(__proxy_t))) {{return;}}
+  __cuda_atomic_bind_load<__proxy_t, __proxy_tag, _Sco, __atomic_cuda_mmio_disable> __bound_load{__ptr_proxy, __dst_proxy};
+  __cuda_atomic_load_memory_order_dispatch(__bound_load, __memorder, _Sco{});
+}
+)XXX";
+}
+
+inline void FormatStore(std::ostream& out)
+{
+  out << R"XXX(
+template <class _Fn, class _Sco>
+static inline _CCCL_DEVICE void __cuda_atomic_store_memory_order_dispatch(_Fn &__cuda_store, int __memorder, _Sco) {
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (
+      switch (__memorder) {
+        case __ATOMIC_RELEASE: __cuda_store(__atomic_cuda_release{}); break;
+        case __ATOMIC_SEQ_CST: __cuda_atomic_fence(_Sco{}, __atomic_cuda_seq_cst{}); [[fallthrough]];
+        case __ATOMIC_RELAXED: __cuda_store(__atomic_cuda_relaxed{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    ),
+    NV_IS_DEVICE, (
+      switch (__memorder) {
+        case __ATOMIC_RELEASE: [[fallthrough]];
+        case __ATOMIC_SEQ_CST: __cuda_atomic_membar(_Sco{}); [[fallthrough]];
+        case __ATOMIC_RELAXED: __cuda_store(__atomic_cuda_volatile{}); break;
+        default: _CCCL_ASSERT(false, "invalid memory order");
+      }
+    )
+  )
+}
+)XXX";
+  // Argument ID Reference
+  // 0 - Operand Type
+  // 1 - Operand Size
+  // 2 - Constraint
+  // 3 - Memory order
+  // 4 - Memory order semantic
+  // 5 - Scope tag
+  // 6 - Scope semantic
+  // 7 - Mmio tag
+  // 8 - Mmio semantic
+  constexpr auto asm_intrinsic_format_128 = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_store(
+  _Type* __ptr, _Type& __val, {3}, __atomic_cuda_operand_{0}{1}, {5}, {7})
+{{
+  static_assert(__cccl_ptx_isa >= 840 && (sizeof(_Type) == 16), "128b ld/st is not supported until PTX ISA version 840");
+  NV_DISPATCH_TARGET(
+    NV_PROVIDES_SM_70, (),
+    NV_ANY_TARGET, (__atomic_ldst_128b_unsupported_before_SM_70();)
+  )
+  asm volatile(R"YYY(
+    {{
+      .reg .b128 _v;
+      mov.b128 _v, {{%1, %2}};
+      st{8}{4}{6}.b128 [%0],_v;
+    }}
+  )YYY" :: "l"(__ptr), "l"(__val.__x),"l"(__val.__y) : "memory");
+}})XXX";
+  constexpr auto asm_intrinsic_format     = R"XXX(
+template <class _Type>
+static inline _CCCL_DEVICE void __cuda_atomic_store(
+  _Type* __ptr, _Type& __val, {3}, __atomic_cuda_operand_{0}{1}, {5}, {7})
+{{ asm volatile("st{8}{4}{6}.{0}{1} [%0],%1;" :: "l"(__ptr), "{2}"(__val) : "memory"); }})XXX";
+
+  constexpr size_t supported_sizes[] = {
+    16,
+    32,
+    64,
+    128,
+  };
+
+  constexpr Operand supported_types[] = {
+    Operand::Bit,
+  };
+
+  constexpr Semantic supported_semantics[] = {
+    Semantic::Release,
+    Semantic::Relaxed,
+    Semantic::Volatile,
+  };
+
+  constexpr Scope supported_scopes[] = {
+    Scope::CTA,
+    Scope::Cluster,
+    Scope::GPU,
+    Scope::System,
+  };
+
+  constexpr Mmio mmio_states[] = {
+    Mmio::Disabled,
+    Mmio::Enabled,
+  };
+
+  for (auto size : supported_sizes)
+  {
+    for (auto type : supported_types)
+    {
+      for (auto sem : supported_semantics)
+      {
+        for (auto sco : supported_scopes)
+        {
+          for (auto mm : mmio_states)
+          {
+            if (size == 16 && type == Operand::Floating)
+            {
+              continue;
+            }
+            if (size == 128 && type != Operand::Bit)
+            {
+              continue;
+            }
+            if ((mm == Mmio::Enabled) && ((sco != Scope::System) || (sem != Semantic::Relaxed)))
+            {
+              continue;
+            }
+
+            if (size == 128)
+            {
+              out << std::format(
+                asm_intrinsic_format_128,
+                /* 0 */ operand(type),
+                /* 1 */ size,
+                /* 2 */ constraints(type, size),
+                /* 3 */ semantic_tag(sem),
+                /* 4 */ semantic_ld_st(sem),
+                /* 5 */ scope_tag(sco),
+                /* 6 */ scope_ld_st(sem, sco),
+                /* 7 */ mmio_tag(mm),
+                /* 8 */ mmio(mm));
+            }
+            else
+            {
+              out << std::format(
+                asm_intrinsic_format,
+                /* 0 */ operand(type),
+                /* 1 */ size,
+                /* 2 */ constraints(type, size),
+                /* 3 */ semantic_tag(sem),
+                /* 4 */ semantic_ld_st(sem),
+                /* 5 */ scope_tag(sco),
+                /* 6 */ scope_ld_st(sem, sco),
+                /* 7 */ mmio_tag(mm),
+                /* 8 */ mmio(mm));
+            }
+          }
+        }
+      }
+    }
+  }
+  out << "\n"
+      << R"XXX(
+template <typename _Type, typename _Tag, typename _Sco, typename _Mmio>
+struct __cuda_atomic_bind_store {
+  _Type* __ptr;
+  _Type* __val;
+
+  template <typename _Atomic_Memorder>
+  inline _CCCL_DEVICE void operator()(_Atomic_Memorder) {
+    __cuda_atomic_store(__ptr, *__val, _Atomic_Memorder{}, _Tag{}, _Sco{}, _Mmio{});
+  }
+};
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE void __atomic_store_cuda(_Type* __ptr, _Type& __val, int __memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(__ptr);
+  __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__val);
+  if (__cuda_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {{return;}}
+  __cuda_atomic_bind_store<__proxy_t, __proxy_tag, _Sco, __atomic_cuda_mmio_disable> __bound_store{__ptr_proxy, __val_proxy};
+  __cuda_atomic_store_memory_order_dispatch(__bound_store, __memorder, _Sco{});
+}
+template <class _Type, class _Sco>
+static inline _CCCL_DEVICE void __atomic_store_cuda(volatile _Type* __ptr, _Type& __val, int __memorder, _Sco)
+{
+  using __proxy_t        = typename __atomic_cuda_deduce_bitwise<_Type>::__type;
+  using __proxy_tag      = typename __atomic_cuda_deduce_bitwise<_Type>::__tag;
+  __proxy_t* __ptr_proxy = reinterpret_cast<__proxy_t*>(const_cast<_Type*>(__ptr));
+  __proxy_t* __val_proxy = reinterpret_cast<__proxy_t*>(&__val);
+  if (__cuda_store_weak_if_local(__ptr_proxy, __val_proxy, sizeof(__proxy_t))) {{return;}}
+  __cuda_atomic_bind_store<__proxy_t, __proxy_tag, _Sco, __atomic_cuda_mmio_disable> __bound_store{__ptr_proxy, __val_proxy};
+  __cuda_atomic_store_memory_order_dispatch(__bound_store, __memorder, _Sco{});
+}
+)XXX";
+}
+
+#endif // LD_ST_H