[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/libcudacxx/test/atomic_codegen/CMakeLists.txt

@@ -0,0 +1,67 @@
+add_custom_target(libcudacxx.test.atomics.ptx)
+
+find_program(filecheck "FileCheck")
+
+if (filecheck)
+  message("-- ${filecheck} found... building atomic codegen tests")
+else()
+  return()
+endif()
+
+find_program(cuobjdump "cuobjdump" REQUIRED)
+find_program(bash "bash" REQUIRED)
+
+set(atomic_codegen_cuda_arch 80)
+
+set(libcudacxx_atomic_codegen_tests)
+if (NOT "NVHPC" STREQUAL "${CMAKE_CXX_COMPILER_ID}")
+  file(GLOB libcudacxx_atomic_codegen_tests "*.cu")
+endif()
+
+# For every atomic API compile the TU and check if the SASS/PTX matches the expected result
+foreach (test_path IN LISTS libcudacxx_atomic_codegen_tests)
+  cmake_path(GET test_path FILENAME test_file)
+  cmake_path(REMOVE_EXTENSION test_file LAST_ONLY OUTPUT_VARIABLE test_name)
+
+  add_library(atomic_codegen_${test_name} STATIC "${test_path}")
+
+  set_target_properties(
+    atomic_codegen_${test_name}
+    PROPERTIES
+      CUDA_ARCHITECTURES "${atomic_codegen_cuda_arch}"
+      COMPILE_DEFINITIONS "_CCCL_ATOMIC_UNSAFE_AUTOMATIC_STORAGE=1"
+  )
+
+  # Clang stopped emitting PTX in clang20. Add flags to re-enable it.
+  if (
+    CMAKE_CUDA_COMPILER_ID STREQUAL Clang
+    AND CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 20
+  )
+    target_compile_options(
+      atomic_codegen_${test_name}
+      PRIVATE "--cuda-include-ptx=sm_${atomic_codegen_cuda_arch}"
+    )
+  endif()
+
+  target_compile_options(atomic_codegen_${test_name} PRIVATE "-Wno-comment")
+
+  ## Important for testing the local headers
+  target_include_directories(
+    atomic_codegen_${test_name}
+    PRIVATE "${libcudacxx_SOURCE_DIR}/include"
+  )
+  add_dependencies(libcudacxx.test.atomics.ptx atomic_codegen_${test_name})
+
+  # Add output path to object directory
+  add_custom_command(
+    TARGET libcudacxx.test.atomics.ptx
+    POST_BUILD
+    # gersemi: off
+    COMMAND
+      "${CMAKE_CURRENT_SOURCE_DIR}/dump_and_check.bash"
+        $<TARGET_FILE:atomic_codegen_${test_name}>
+        "${test_path}"
+        SM8X
+    # gersemi: on
+  )
+endforeach()

cccl_upstream/libcudacxx/test/atomic_codegen/atomic_add_non_volatile.cu

@@ -0,0 +1,21 @@
+#include <cuda/atomic>
+
+__global__ void add_relaxed_device_non_volatile(int* data, int* out, int n)
+{
+  auto ref = cuda::atomic_ref<int, cuda::thread_scope_device>{*(data)};
+  *out     = ref.fetch_add(n, cuda::std::memory_order_relaxed);
+}
+
+/*
+
+; SM8X-LABEL: .target sm_80
+; SM8X:      .visible .entry [[FUNCTION:_.*add_relaxed_device_non_volatile.*]](
+; SM8X-DAG:  ld.param.{{b|u}}64 %rd[[#ATOM:]], {{.*}}[[FUNCTION]]_param_0{{.*}}
+; SM8X-DAG:  ld.param.{{b|u}}64 %rd[[#RESULT:]], {{.*}}[[FUNCTION]]_param_1{{.*}}
+; SM8X-DAG:  ld.param.{{b|u}}32 %r[[#INPUT:]], {{.*}}[[FUNCTION]]_param_2{{.*}}
+; SM8X-DAG:  cvta.to.global.u64 %rd[[#GOUT:]], %rd[[#RESULT]];
+; SM8X-NEXT: {{/*[[:space:]] *}}atom.add.relaxed.gpu.s32 %r[[#DEST:]],[%rd[[#ATOM]]],%r[[#INPUT]];{{[[:space:]]/*}}
+; SM8X-NEXT: st.global.{{b|u}}32 [%rd[[#GOUT]]], %r[[#DEST]];
+; SM8X-NEXT: ret;
+
+*/

cccl_upstream/libcudacxx/test/atomic_codegen/atomic_cas_non_volatile.cu

@@ -0,0 +1,23 @@
+#include <cuda/atomic>
+
+__global__ void cas_device_relaxed_non_volatile(int* data, int* out, int n)
+{
+  auto ref = cuda::atomic_ref<int, cuda::thread_scope_device>{*(data)};
+  ref.compare_exchange_strong(*out, n, cuda::std::memory_order_relaxed);
+}
+
+// clang-format off
+/*
+
+; SM8X-LABEL: .target sm_80
+; SM8X:      .visible .entry [[FUNCTION:_.*cas_device_relaxed_non_volatile.*]](
+; SM8X-DAG:  ld.param.{{b|u}}64 %rd[[#ATOM:]], {{.*}}[[FUNCTION]]_param_0{{.*}}
+; SM8X-DAG:  ld.param.{{b|u}}64 %rd[[#EXPECTED:]], {{.*}}[[FUNCTION]]_param_1{{.*}}
+; SM8X-DAG:  ld.param.{{b|u}}32 %r[[#INPUT:]], {{.*}}[[FUNCTION]]_param_2{{.*}}
+; SM8X-DAG:  cvta.to.global.u64 %rd[[#GOUT:]], %rd[[#EXPECTED]];
+; SM8X-DAG:  ld.global.{{b|u}}32 %r[[#LOCALEXP:]], [%rd[[#INPUT]]];
+; SM8X-NEXT: {{/*[[:space:]] *}}atom.cas.relaxed.gpu.b32 %r[[#DEST:]],[%rd[[#ATOM]]],%r[[#LOCALEXP]],%r[[#INPUT]];{{[[:space:]]/*}}
+; SM8X-NEXT: st.global.{{b|u}}32 [%rd[[#GOUT]]], %r[[#DEST]];
+; SM8X-NEXT: ret;
+
+*/

cccl_upstream/libcudacxx/test/atomic_codegen/atomic_exch_non_volatile.cu

@@ -0,0 +1,21 @@
+#include <cuda/atomic>
+
+__global__ void exch_device_relaxed_non_volatile(int* data, int* out, int n)
+{
+  auto ref = cuda::atomic_ref<int, cuda::thread_scope_device>{*(data)};
+  *out     = ref.exchange(n, cuda::std::memory_order_relaxed);
+}
+
+/*
+
+; SM8X-LABEL: .target sm_80
+; SM8X:      .visible .entry [[FUNCTION:_.*exch_device_relaxed_non_volatile.*]](
+; SM8X-DAG:  ld.param.{{b|u}}64 %rd[[#ATOM:]], {{.*}}[[FUNCTION]]_param_0{{.*}}
+; SM8X-DAG:  ld.param.{{b|u}}64 %rd[[#EXPECTED:]], {{.*}}[[FUNCTION]]_param_1{{.*}}
+; SM8X-DAG:  ld.param.{{b|u}}32 %r[[#INPUT:]], {{.*}}[[FUNCTION]]_param_2{{.*}}
+; SM8X-DAG:  cvta.to.global.u64 %rd[[#GOUT:]], %rd[[#EXPECTED]];
+; SM8X-NEXT: {{/*[[:space:]] *}}atom.exch.relaxed.gpu.b32 %r[[#DEST:]],[%rd[[#ATOM]]],%r[[#INPUT]];{{[[:space:]]/*}}
+; SM8X-NEXT: st.global.{{b|u}}32 [%rd[[#GOUT]]], %r[[#DEST]];
+; SM8X-NEXT: ret;
+
+*/

cccl_upstream/libcudacxx/test/atomic_codegen/atomic_load_non_volatile.cu

@@ -0,0 +1,20 @@
+#include <cuda/atomic>
+
+__global__ void load_relaxed_device_non_volatile(int* data, int* out)
+{
+  auto ref = cuda::atomic_ref<int, cuda::thread_scope_device>{*(data)};
+  *out     = ref.load(cuda::std::memory_order_relaxed);
+}
+
+/*
+
+; SM8X-LABEL: .target sm_80
+; SM8X:      .visible .entry [[FUNCTION:_.*load_relaxed_device_non_volatile.*]](
+; SM8X-DAG:  ld.param.{{b|u}}64 %rd[[#ATOM:]], {{.*}}[[FUNCTION]]_param_0{{.*}}
+; SM8X-DAG:  ld.param.{{b|u}}64 %rd[[#EXPECTED:]], {{.*}}[[FUNCTION]]_param_1{{.*}}
+; SM8X-DAG:  cvta.to.global.u64 %rd[[#GOUT:]], %rd[[#EXPECTED]];
+; SM8X-NEXT: {{/*[[:space:]] *}}ld.relaxed.gpu.b32 %r[[#DEST:]],[%rd[[#ATOM]]];{{[[:space:]]/*}}
+; SM8X-NEXT: st.global.{{b|u}}32 [%rd[[#GOUT]]], %r[[#DEST]];
+; SM8X-NEXT: ret;
+
+*/

cccl_upstream/libcudacxx/test/atomic_codegen/atomic_store_non_volatile.cu

@@ -0,0 +1,18 @@
+#include <cuda/atomic>
+
+__global__ void store_relaxed_device_non_volatile(int* data, int in)
+{
+  auto ref = cuda::atomic_ref<int, cuda::thread_scope_device>{*(data)};
+  ref.store(in, cuda::std::memory_order_relaxed);
+}
+
+/*
+
+; SM8X-LABEL: .target sm_80
+; SM8X:      .visible .entry [[FUNCTION:_.*store_relaxed_device_non_volatile.*]](
+; SM8X-DAG:  ld.param.{{b|u}}64 %rd[[#ATOM:]], {{.*}}[[FUNCTION]]_param_0{{.*}}
+; SM8X-DAG:  ld.param.{{b|u}}32 %r[[#INPUT:]], {{.*}}[[FUNCTION]]_param_1{{.*}}
+; SM8X-NEXT: {{/*[[:space:]] *}}st.relaxed.gpu.b32 [%rd[[#ATOM]]],%r[[#INPUT]];{{[[:space:]]/*}}
+; SM8X-NEXT: ret;
+
+*/

cccl_upstream/libcudacxx/test/atomic_codegen/atomic_sub_non_volatile.cu

@@ -0,0 +1,22 @@
+#include <cuda/atomic>
+
+__global__ void sub_relaxed_device_non_volatile(int* data, int* out, int n)
+{
+  auto ref = cuda::atomic_ref<int, cuda::thread_scope_device>{*(data)};
+  *out     = ref.fetch_sub(n, cuda::std::memory_order_relaxed);
+}
+
+/*
+
+; SM8X-LABEL: .target sm_80
+; SM8X:      .visible .entry [[FUNCTION:_.*sub_relaxed_device_non_volatile.*]](
+; SM8X-DAG:  ld.param.{{b|u}}64 %rd[[#ATOM:]], {{.*}}[[FUNCTION]]_param_0{{.*}}
+; SM8X-DAG:  ld.param.{{b|u}}64 %rd[[#RESULT:]], {{.*}}[[FUNCTION]]_param_1{{.*}}
+; SM8X-DAG:  ld.param.{{b|u}}32 %r[[#INPUT:]], {{.*}}[[FUNCTION]]_param_2{{.*}}
+; SM8X-DAG:  cvta.to.global.u64 %rd[[#GOUT:]], %rd[[#RESULT]];
+; SM8X-NEXT: neg.s32 %r[[#NEG:]], %r[[#INPUT]];
+; SM8X-NEXT: {{/*[[:space:]] *}}atom.add.relaxed.gpu.s32 %r[[#DEST:]],[%rd[[#ATOM]]],%r[[#NEG]];{{[[:space:]]/*}}
+; SM8X-NEXT: st.global.{{b|u}}32 [%rd[[#GOUT]]], %r[[#DEST]];
+; SM8X-NEXT: ret;
+
+*/

cccl_upstream/libcudacxx/test/atomic_codegen/dump_and_check.bash

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+## Usage: dump_and_check test.a test.cu PREFIXES [cuobjdump-mode]
+input_archive="${1}"
+input_testfile="${2}"
+input_prefix="${3}"
+dump_mode="${4:---dump-ptx}"
+filecheck="${FILECHECK:-FileCheck}"
+
+cuobjdump "${dump_mode}" "${input_archive}" | "${filecheck}" --match-full-lines --check-prefixes="${input_prefix}" "${input_testfile}"