[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/cub/benchmarks/bench/scan/exclusive/base.cuh

@@ -0,0 +1,88 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#pragma once
+
+#include <cub/device/device_scan.cuh>
+
+#include <cuda/std/__functional/invoke.h>
+
+#include <nvbench_helper.cuh>
+
+#include "../policy_selector.h"
+
+template <typename T, typename OffsetT>
+static void basic(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+try
+{
+  using init_value_t             = T;
+  using accum_t [[maybe_unused]] = ::cuda::std::__accumulator_t<op_t, init_value_t, T>;
+  using offset_t                 = cub::detail::choose_offset_t<OffsetT>;
+#if USES_LOOKAHEAD()
+  static_assert(sizeof(offset_t) == sizeof(size_t)); // lookahead scan uses size_t internally
+#endif // USES_LOOKAHEAD()
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  if (sizeof(offset_t) == 4 && elements > std::numeric_limits<offset_t>::max())
+  {
+    state.skip("Skipping: input size exceeds 32-bit offset type capacity.");
+    return;
+  }
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements);
+
+  const T* d_input = thrust::raw_pointer_cast(input.data());
+  T* d_output      = thrust::raw_pointer_cast(output.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::ExclusiveScan,
+      "ExclusiveScan failed",
+      d_input,
+      d_output,
+      op_t{},
+      init_value_t{},
+      static_cast<offset_t>(input.size()),
+      env);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+// __half and __nv_bfloat16 are added for full (non-tuning) runs; CUB has fast paths for them (see #9587).
+#ifdef TUNE_T
+using value_types = nvbench::type_list<TUNE_T>;
+#else
+using value_types =
+  push_back_t<all_types
+#  if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
+              ,
+              __half
+#  endif
+#  if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
+              ,
+              __nv_bfloat16
+#  endif
+              >;
+#endif
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(value_types, scan_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/scan/exclusive/by_key.cu

@@ -0,0 +1,106 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_scan.cuh>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#if !TUNE_BASE
+struct bench_scan_by_key_policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ScanByKeyPolicy
+  {
+    return {cub::ScanByKeyAlgorithm::lookback,
+            {TUNE_THREADS,
+             TUNE_ITEMS,
+             TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE,
+             TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : cub::LOAD_CA,
+             TUNE_TRANSPOSE == 0 ? cub::BLOCK_STORE_DIRECT : cub::BLOCK_STORE_WARP_TRANSPOSE,
+             cub::BLOCK_SCAN_WARP_SCANS,
+             lookback_delay_policy}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename KeyT, typename ValueT, typename OffsetT>
+static void scan(nvbench::state& state, nvbench::type_list<KeyT, ValueT, OffsetT>)
+{
+  using init_value_t  = ValueT;
+  using op_t          = ::cuda::std::plus<>;
+  using equality_op_t = ::cuda::std::equal_to<>;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<ValueT> in_vals(elements);
+  thrust::device_vector<ValueT> out_vals(elements);
+  thrust::device_vector<KeyT> keys = generate.uniform.key_segments(elements, 0, 5200);
+
+  const KeyT* d_keys      = thrust::raw_pointer_cast(keys.data());
+  const ValueT* d_in_vals = thrust::raw_pointer_cast(in_vals.data());
+  ValueT* d_out_vals      = thrust::raw_pointer_cast(out_vals.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<ValueT>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_scan_by_key_policy_selector{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::ExclusiveScanByKey,
+      "ExclusiveScanByKey failed",
+      d_keys,
+      d_in_vals,
+      d_out_vals,
+      op_t{},
+      init_value_t{},
+      static_cast<OffsetT>(elements),
+      equality_op_t{},
+      env);
+  });
+}
+
+using some_offset_types = nvbench::type_list<nvbench::int32_t>;
+
+#ifdef TUNE_KeyT
+using key_types = nvbench::type_list<TUNE_KeyT>;
+#else // !defined(TUNE_KeyT)
+using key_types = all_types;
+#endif // TUNE_KeyT
+
+#ifdef TUNE_ValueT
+using value_types = nvbench::type_list<TUNE_ValueT>;
+#else // !defined(TUNE_ValueT)
+using value_types =
+  nvbench::type_list<int8_t,
+                     int16_t,
+                     int32_t,
+                     int64_t
+#  if _CCCL_HAS_INT128()
+                     ,
+                     int128_t
+#  endif
+                     >;
+#endif // TUNE_ValueT
+
+NVBENCH_BENCH_TYPES(scan, NVBENCH_TYPE_AXES(key_types, value_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/scan/exclusive/custom.cu

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+// This benchmark uses a custom operation, max_t, which is not known to CUB, so no operator specific optimizations and
+// tunings are performed.
+
+// Because CUB cannot detect this operator, we cannot add any tunings based on the results of this benchmark. Its main
+// use is to detect regressions.
+
+#include <nvbench_helper.cuh>
+
+#define USES_LOOKAHEAD() 0
+using op_t              = max_t;
+using scan_offset_types = offset_types;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/scan/exclusive/deterministic.cu

@@ -0,0 +1,57 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/device/device_scan.cuh>
+
+#include <cuda/__execution/determinism.h>
+#include <cuda/__execution/require.h>
+#include <cuda/std/__functional/invoke.h>
+
+#include <nvbench_helper.cuh>
+
+template <typename T, typename OffsetT>
+static void exclusive_scan(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+try
+{
+  using init_value_t = T;
+  using offset_t     = OffsetT;
+  using scan_op_t    = ::cuda::std::plus<T>;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements, thrust::no_init);
+
+  const T* d_input = thrust::raw_pointer_cast(input.data());
+  T* d_output      = thrust::raw_pointer_cast(output.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(alloc, launch, cuda::execution::require(cuda::execution::determinism::run_to_run));
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::ExclusiveScan,
+      "ExclusiveScan failed",
+      d_input,
+      d_output,
+      scan_op_t{},
+      init_value_t{},
+      static_cast<offset_t>(elements),
+      env);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+using types   = nvbench::type_list<float, double>;
+using offsets = nvbench::type_list<int64_t>;
+
+NVBENCH_BENCH_TYPES(exclusive_scan, NVBENCH_TYPE_AXES(types, offsets))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/scan/exclusive/sum.cu

@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+// Tuning parameters found for signed integer types apply equally for unsigned integer types
+
+#include <nvbench_helper.cuh>
+
+// This benchmark tunes the old, non-lookahead scan implementation. Using it for benchmarking, will pick the lookahead
+// implementation on SM100+, but it's better to use the sum.lookahead.cu benchmark instead, which uses a single OffsetT.
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#define USES_LOOKAHEAD() 0
+using op_t              = ::cuda::std::plus<>;
+using scan_offset_types = offset_types;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/scan/exclusive/sum.lookahead.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// This tunes the lookahead implementation of scan, which is only available on SM100+. It has entirely different tuning
+// parameters and is agnostic of the offset type. It is thus in a separate file, so we can continue to tune the old scan
+// implementation on older hardware architectures.
+
+#include <cuda/__cccl_config>
+
+#if _CCCL_PP_COUNT(__CUDA_ARCH_LIST__) != 1
+#  warning "This benchmark does not support being compiled for multiple architectures. Disabling it."
+#else // _CCCL_PP_COUNT(__CUDA_ARCH_LIST__) != 1
+
+#  if __CUDA_ARCH_LIST__ < 1000
+// We don't care if clang-tidy can't parse this
+#    ifndef _CCCL_CLANG_TIDY_INVOKED
+#      warning "Lookahead scan requires at least sm_100. Disabling it."
+#    endif // !defined _CCCL_CLANG_TIDY_INVOKED
+#  else // __CUDA_ARCH_LIST__ < 1000
+
+#    if __cccl_ptx_isa < 860
+#      warning "Lookahead scan requires at least PTX ISA 8.6. Disabling it."
+#    else // if __cccl_ptx_isa < 860
+
+#      include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_NUM_REDUCE_SCAN_WARPS wrps 1:8:1
+// %RANGE% TUNE_NUM_LOOKBACK_ITEMS lbi 1:8:1
+
+// TODO(bgruber): find a good range and step width, items per thread should be coprime with 32 to avoid SMEM conflicts.
+// Should we specify nominal items per thread instead?
+// %RANGE% TUNE_ITEMS_PLUS_ONE ipt 8:256:8
+
+// %RANGE% TUNE_LOOKBACK_STAGES lbs -2:2:1
+// %RANGE% TUNE_BLOCK_IDX_STAGES bis -2:2:1
+
+#      define USES_LOOKAHEAD() 1
+using op_t              = ::cuda::std::plus<>;
+using scan_offset_types = nvbench::type_list<int64_t>;
+#      include "base.cuh"
+
+#    endif // __cccl_ptx_isa < 860
+#  endif // __CUDA_ARCH_LIST__ < 1000
+#endif // _CCCL_PP_COUNT(__CUDA_ARCH_LIST__) != 1