[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/cub/benchmarks/bench/scan/applications/P1/log-cdf-from-log-pdfs.cu

@@ -0,0 +1,170 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/device_scan.cuh>
+
+#include <thrust/device_vector.h>
+#include <thrust/host_vector.h>
+
+#include <cuda/std/cmath>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_DIRECT
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_WARP_TRANSPOSE
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+#endif // !TUNE_BASE
+
+#include "../../policy_selector.h"
+
+namespace impl
+{
+/*
+ * Given a sequence of logarithms of probability mass function values,
+ * compute sequence of logarithms of cumulative distribution function values.
+ *
+ *    log(CDF(n)) = log(\sum( PDF(k), 0 <=k <=n ))
+ *
+ * This is inclusive scan using logaddexp binary operator:
+ *    logaddexp( logpdf1, logpdf2 ) := log( exp(logpdf1) + exp(logpdf2) )
+ *         == max(logpdf1, logpdf2) + log( 1 + exp(-abs(logpdf1 - logpdf2)))
+ *
+ * The last reformulation allows avoid numerical accuracy issues
+ * caused by underflows.
+ *
+ */
+
+struct log_add_plus
+{
+  /* Operator is commutative and associative */
+  template <typename T>
+  T __host__ __device__ operator()(T v1, T v2)
+  {
+    T max12 = cuda::maximum{}(v1, v2);
+    T min12 = cuda::minimum{}(v1, v2);
+    T exp   = cuda::std::exp(min12 - max12);
+    return max12 + cuda::std::log1p(exp);
+  }
+};
+
+template <typename T>
+struct log_pdf_builder
+{
+  T mu;
+  T norm;
+  cuda::std::size_t n;
+
+  T __host__ __device__ operator()(cuda::std::size_t i) const
+  {
+    return -mu * static_cast<T>(n - i) + norm;
+  }
+};
+
+template <typename T>
+[[nodiscard]] bool validate(const thrust::device_vector<T>& output, cudaStream_t stream)
+{
+  cudaStreamSynchronize(stream);
+
+  thrust::host_vector<T> h_output(output);
+  auto elements = h_output.size();
+  // test is designed so that last element of prefix scan sequence should be close to log(1.0) == 0.0
+  bool check = cuda::std::abs(h_output[elements - 1])
+             < cuda::std::sqrt(static_cast<T>(1 + elements)) * cuda::std::numeric_limits<T>::epsilon();
+
+  return check;
+}
+}; // namespace impl
+
+template <typename FloatingPointT, typename OffsetT>
+static void inclusive_scan(nvbench::state& state, nvbench::type_list<FloatingPointT, OffsetT>)
+{
+  static_assert(cuda::std::is_floating_point_v<FloatingPointT>);
+
+  using value_t                  = FloatingPointT;
+  using input_t                  = const value_t*;
+  using output_t                 = value_t*;
+  using op_t                     = impl::log_add_plus;
+  using accum_t [[maybe_unused]] = value_t;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  auto mu             = static_cast<value_t>(state.get_float64("Mu{io}"));
+
+  auto norm = cuda::std::log1p(-cuda::std::exp(-mu)) - cuda::std::log1p(-cuda::std::exp(-mu * elements));
+
+  thrust::device_vector<value_t> input(elements, thrust::no_init);
+
+  cudaStream_t bench_stream = state.get_cuda_stream();
+
+  auto naturals_it = cuda::counting_iterator(cuda::std::size_t{0});
+  cub::DeviceTransform::Transform(
+    cuda::std::make_tuple(naturals_it),
+    input.begin(),
+    elements,
+    impl::log_pdf_builder<value_t>{mu, norm, elements},
+    bench_stream);
+
+  thrust::device_vector<value_t> output(elements, thrust::no_init);
+
+  input_t d_input   = thrust::raw_pointer_cast(input.data());
+  output_t d_output = thrust::raw_pointer_cast(output.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<value_t>(elements, "Size");
+  state.add_global_memory_writes<value_t>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::InclusiveScan,
+      "InclusiveScan failed",
+      d_input,
+      d_output,
+      op_t{},
+      static_cast<OffsetT>(input.size()),
+      env);
+  });
+
+  // for validation, use
+  // assert(impl::validate(output, bench_stream));
+}
+
+#ifdef TUNE_T
+using fp_types = nvbench::type_list<TUNE_T>;
+#else
+using fp_types = nvbench::type_list<float, double>;
+#endif
+
+NVBENCH_BENCH_TYPES(inclusive_scan, NVBENCH_TYPE_AXES(fp_types, offset_types))
+  .set_name("app-logcdf-from-logpdf")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_float64_axis("Mu{io}", {1e-4f});

cccl_upstream/cub/benchmarks/bench/scan/applications/P1/non-commutative-bicyclic-monoid.cu

@@ -0,0 +1,153 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/device_scan.cuh>
+#include <cub/device/device_transform.cuh>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_DIRECT
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_WARP_TRANSPOSE
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+#endif // !TUNE_BASE
+
+#include "../../policy_selector.h"
+
+namespace impl
+{
+/* Consider free monoid with two generators, ``q`` and ``p``, modulo defining relationship (``p * q == 1``).
+ * Elements of this algebra are ``q^m * p^n``, identified by a pair of integral exponents. The identity
+ * element is ``1 == q^0 * p^0``, which maps to pair of zeros ``e = (0, 0)``.
+ *
+ * The product is defined by concatenation:
+ *     q^m * p^n * q^r * p^s == q^m * p^{n-1} * p * q * q^{r-1} * p^s
+ *                           == q^m * p^{n-1} * q^{r-1} * p^s
+ *
+ * This reduction can be performed ``min(n, r)`` times resulting in
+ *
+ *     q^m * p^n * q^r * p^s == q^{m + r - min(n, r)} * p^{s + n - min(n, r)}
+ *
+ * Hence this is a monoid, known as bicyclic monoid.
+ * This operation of pairs of integers is associative (since concatenation is), but non-commutative.
+ *
+ *    Ref: https://en.wikipedia.org/wiki/Bicyclic_semigroup
+ *    Ref: https://en.wikipedia.org/wiki/Monoid
+ */
+
+template <typename UnsignedIntegralT>
+struct bicyclic_monoid_op
+{
+  static_assert(cuda::std::is_integral_v<UnsignedIntegralT>);
+  static_assert(cuda::std::is_unsigned_v<UnsignedIntegralT>);
+
+  using pair_t = cuda::std::pair<UnsignedIntegralT, UnsignedIntegralT>;
+  using min_t  = cuda::minimum<>;
+
+  // Operator is associative but non-commutative
+  pair_t __host__ __device__ operator()(pair_t v1, pair_t v2) const
+  {
+    auto [m, n] = v1;
+    auto [r, s] = v2;
+    auto min_nr = min_t{}(n, r);
+    return {m + r - min_nr, s + n - min_nr};
+  }
+};
+
+template <typename T>
+struct repack_pair
+{
+  cuda::std::pair<T, T> __host__ __device__ operator()(const T& v1, const T& v2) const
+  {
+    return {v1, v2};
+  };
+};
+}; // namespace impl
+
+template <typename T, typename OffsetT>
+static void inclusive_scan(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  static_assert(cuda::std::is_integral_v<T> && cuda::std::is_unsigned_v<T>, "Unsigned integral type should be used");
+  using pair_t                   = cuda::std::pair<T, T>;
+  using op_t                     = impl::bicyclic_monoid_op<T>;
+  using accum_t [[maybe_unused]] = pair_t;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<pair_t> output(elements);
+
+  thrust::device_vector<pair_t> input(elements);
+  {
+    thrust::device_vector<T> q_exponents = generate(elements);
+    thrust::device_vector<T> p_exponents = generate(elements);
+
+    impl::repack_pair<T> repack_op{};
+
+    cub::DeviceTransform::Transform(
+      cuda::std::tuple{q_exponents.begin(), p_exponents.begin()}, input.begin(), elements, repack_op);
+
+    // deallocate temporary arrays at the scope boundary
+  }
+
+  pair_t* d_input  = thrust::raw_pointer_cast(input.data());
+  pair_t* d_output = thrust::raw_pointer_cast(output.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<pair_t>(elements, "Size");
+  state.add_global_memory_writes<pair_t>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::InclusiveScan,
+      "InclusiveScan failed",
+      d_input,
+      d_output,
+      op_t{},
+      static_cast<OffsetT>(input.size()),
+      env);
+  });
+}
+
+#ifdef TUNE_T
+using uint_types = nvbench::type_list<TUNE_T>;
+#else
+#  if _CCCL_HAS_INT128()
+using uint_types = nvbench::type_list<cuda::std::uint32_t, cuda::std::uint64_t, uint128_t>;
+#  else
+using uint_types = nvbench::type_list<cuda::std::uint32_t, cuda::std::uint64_t>;
+#  endif
+#endif
+
+NVBENCH_BENCH_TYPES(inclusive_scan, NVBENCH_TYPE_AXES(uint_types, offset_types))
+  .set_name("app-bicyclic-monoid")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/scan/applications/P1/rabin-karp-second-fingerprinting.cu

@@ -0,0 +1,347 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/device_scan.cuh>
+
+#include <thrust/host_vector.h>
+
+#include <cuda/cmath>
+#include <cuda/std/limits>
+
+#include <iostream>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_DIRECT
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_WARP_TRANSPOSE
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+#endif // !TUNE_BASE
+
+#include "../../policy_selector.h"
+
+namespace impl
+{
+/* Denote epsilon, the identity element, be an empty sequence, and consider
+ * set of sequences of {0, 1} bits, with binary operation of concatenation.
+ *
+ * Define homomorphism K from the set of sequence to 2-by-2 integral matrices
+ * over cyclic ring Z_p for some prime p.
+ *
+ *    K( ''  ) = [[ 1, 0], [0, 1]]
+ *    K( '0' ) = [[1, 0], [1, 1]]
+ *    K( '1' ) = [[1, 1], [0, 1]]
+ *
+ *    K( concat(seq1, seq2) ) := matmul( K(seq1), K(seq2) ) in Z_p
+ *
+ * Given a sequence of unsigned integers, encoding bit sequences,
+ * we build transform iterator mapping integer to the matrix. Then
+ * call inclusive_scan with matrix multiply operator in Z_p
+ *
+ * Ref: https://doi.org/10.1147/rd.312.0249
+ */
+
+// Types associated with the cyclic ring
+using ZpT   = cuda::std::uint32_t;
+using WideT = cuda::std::uint64_t;
+
+using MatT = cuda::std::array<ZpT, 4>;
+
+inline ZpT __host__ __device__ Zp_mul(ZpT v1, ZpT v2, cuda::fast_mod_div<WideT> m_p)
+{
+  const auto w1 = static_cast<WideT>(v1);
+  const auto w2 = static_cast<WideT>(v2);
+  return static_cast<ZpT>((w1 * w2) % m_p);
+}
+
+inline ZpT __host__ __device__ Zp_add(ZpT v1, ZpT v2, cuda::fast_mod_div<WideT> m_p)
+{
+  const auto w1 = static_cast<WideT>(v1);
+  const auto w2 = static_cast<WideT>(v2);
+  return static_cast<ZpT>((w1 + w2) % m_p);
+}
+
+inline MatT __host__ __device__ Zp_matmul(MatT v1, MatT v2, cuda::fast_mod_div<WideT> m_p)
+{
+  ZpT _1_00_2_00 = Zp_mul(v1[0], v2[0], m_p);
+  ZpT _1_01_2_10 = Zp_mul(v1[1], v2[2], m_p);
+  ZpT _r_00      = Zp_add(_1_00_2_00, _1_01_2_10, m_p);
+
+  ZpT _1_00_2_01 = Zp_mul(v1[0], v2[1], m_p);
+  ZpT _1_01_2_11 = Zp_mul(v1[1], v2[3], m_p);
+  ZpT _r_01      = Zp_add(_1_00_2_01, _1_01_2_11, m_p);
+
+  ZpT _1_10_2_00 = Zp_mul(v1[2], v2[0], m_p);
+  ZpT _1_11_2_10 = Zp_mul(v1[3], v2[2], m_p);
+  ZpT _r_10      = Zp_add(_1_10_2_00, _1_11_2_10, m_p);
+
+  ZpT _1_10_2_01 = Zp_mul(v1[2], v2[1], m_p);
+  ZpT _1_11_2_11 = Zp_mul(v1[3], v2[3], m_p);
+  ZpT _r_11      = Zp_add(_1_10_2_01, _1_11_2_11, m_p);
+
+  return {_r_00, _r_01, _r_10, _r_11};
+}
+
+struct RabinKarpOp
+{
+  cuda::fast_mod_div<WideT> m_p;
+
+  __host__ __device__ RabinKarpOp(ZpT p)
+      : m_p(static_cast<WideT>(p))
+  {}
+
+  // scan operator: non-commutative and associative
+  MatT __host__ __device__ operator()(MatT v1, MatT v2) const
+  {
+    return Zp_matmul(v1, v2, m_p);
+  }
+};
+
+template <typename T>
+struct ChunkToMat
+{
+  static_assert(cuda::std::is_integral_v<T> && cuda::std::is_unsigned_v<T>, "Bit sequence should be represented");
+
+  cuda::fast_mod_div<WideT> m_p;
+
+  __host__ __device__ ChunkToMat(ZpT p)
+      : m_p(static_cast<WideT>(p))
+  {}
+
+  MatT __host__ __device__ operator()(const T& bits) const
+  {
+    static constexpr int n_bits = cuda::std::numeric_limits<T>::digits;
+    static_assert(n_bits >= 1, "Type must have non-zero bitwidth");
+
+    static constexpr MatT _0 = {ZpT{1}, ZpT{0}, ZpT{1}, ZpT{1}}; // [[1, 0], [1, 1]]
+    static constexpr MatT _1 = {ZpT{1}, ZpT{1}, ZpT{0}, ZpT{1}}; // [[1, 1], [0, 1]]
+
+    // initialize with identity matrix
+    MatT m  = (bits & 1) ? _1 : _0;
+    T _bits = bits >> 1;
+
+    // use of cuda::static_for here results in performance regression due to increased register pressure
+    for (int i = 1; i < n_bits; ++i)
+    {
+      (void) i;
+      m = Zp_matmul((_bits & 1) ? _1 : _0, m, m_p);
+      _bits >>= 1;
+    }
+
+    return m;
+  }
+};
+
+// Iterator that performs assignment at specific index only, discards otherwise
+//
+// This iterator allows tp use inclusive_scan to perform reduction with
+// non-commutative associative binary operator
+//
+template <typename OffsetT, typename Iter>
+struct write_at_specific_index_or_discard
+{
+private:
+  OffsetT m_index{};
+  OffsetT m_target_index;
+  Iter m_iter;
+
+  void __host__ __device__ set_index(OffsetT index)
+  {
+    m_index = index;
+  }
+
+public:
+  struct assign_proxy
+  {
+  private:
+    bool m_writable;
+    Iter m_iter;
+
+  public:
+    __host__ __device__ assign_proxy(bool writable, Iter iter)
+        : m_writable(writable)
+        , m_iter(iter)
+    {}
+
+    template <typename Tp>
+    constexpr assign_proxy& __host__ __device__ operator=(Tp&& v)
+    {
+      if (m_writable)
+      {
+        *m_iter = v;
+      }
+      return *this;
+    }
+  };
+
+  using iterator_concept  = cuda::std::random_access_iterator_tag;
+  using iterator_category = cuda::std::random_access_iterator_tag;
+  using value_type        = cuda::std::iter_value_t<Iter>;
+  using difference_type   = cuda::std::iter_difference_t<Iter>;
+  using pointer           = void;
+  using reference         = void;
+
+  write_at_specific_index_or_discard() = delete;
+  explicit __host__ __device__ write_at_specific_index_or_discard(OffsetT offset, Iter iter)
+      : m_target_index(offset)
+      , m_iter(iter)
+  {}
+
+  write_at_specific_index_or_discard(const write_at_specific_index_or_discard&)            = default;
+  write_at_specific_index_or_discard(write_at_specific_index_or_discard&&)                 = default;
+  write_at_specific_index_or_discard& operator=(const write_at_specific_index_or_discard&) = default;
+  write_at_specific_index_or_discard& operator=(write_at_specific_index_or_discard&&)      = default;
+
+  assign_proxy __host__ __device__ operator[](difference_type n)
+  {
+    return {(m_index + static_cast<OffsetT>(n)) == m_target_index, m_iter};
+  }
+
+  write_at_specific_index_or_discard __host__ __device__ operator+(difference_type n) const
+  {
+    auto r = write_at_specific_index_or_discard(m_target_index, m_iter);
+    r.set_index((m_index + static_cast<OffsetT>(n)));
+    return r;
+  }
+};
+
+template <typename InputT, typename OutputT>
+[[nodiscard]] bool validate(
+  const thrust::device_vector<InputT>& input, const thrust::device_vector<OutputT>& output, ZpT p, cudaStream_t stream)
+{
+  using accum_t = OutputT;
+  using input_t = InputT;
+
+  cudaStreamSynchronize(stream);
+
+  thrust::host_vector<accum_t> h_out(output);
+  thrust::host_vector<input_t> h_inp(input);
+
+  accum_t ref_mat = {1, 0, 0, 1};
+
+  static constexpr accum_t mat_0 = {1, 0, 1, 1}; // lower diagonal
+  static constexpr accum_t mat_1 = {1, 1, 0, 1}; // upper diagonal
+  cuda::fast_mod_div<impl::WideT> mod(p);
+  for (auto&& el : h_inp)
+  {
+    input_t v = el;
+
+    accum_t word_mat = {1, 0, 0, 1};
+    for (int i = 0; i < sizeof(input_t) * 8; ++i)
+    {
+      if (v & 1)
+      {
+        word_mat = impl::Zp_matmul(mat_1, word_mat, mod);
+      }
+      else
+      {
+        word_mat = impl::Zp_matmul(mat_0, word_mat, mod);
+      }
+      v >>= 1;
+    }
+
+    ref_mat = impl::Zp_matmul(ref_mat, word_mat, mod);
+  }
+
+  const accum_t& res = h_out[0];
+  if (ref_mat != res)
+  {
+    std::cout << "FAILED: ";
+    std::cout << "cub_computed([[" << res[0] << ", " << res[1] << "], [" << res[2] << ", " << res[3] << "]]) != ";
+    std::cout
+      << "reference([[" << ref_mat[0] << ", " << ref_mat[1] << "], [" << ref_mat[2] << ", " << ref_mat[3] << "]])\n";
+    return false;
+  }
+
+  return true;
+}
+}; // namespace impl
+
+template <typename BitsetT, typename OffsetT>
+static void inclusive_scan(nvbench::state& state, nvbench::type_list<BitsetT, OffsetT>)
+{
+  using op_t         = impl::RabinKarpOp;
+  using input_t      = BitsetT;
+  using raw_it_t     = const input_t*;
+  using input_it_t   = cuda::transform_iterator<impl::ChunkToMat<input_t>, raw_it_t>;
+  using accum_t      = impl::MatT;
+  using output_ptr_t = impl::MatT*;
+  using output_it_t  = impl::write_at_specific_index_or_discard<OffsetT, output_ptr_t>;
+
+  using ZpT = impl::ZpT;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<input_t> input = generate(elements);
+  thrust::device_vector<accum_t> output(1, thrust::no_init);
+
+  // a large prime
+  ZpT p = static_cast<ZpT>(state.get_int64("Modulus"));
+
+  raw_it_t d_input      = thrust::raw_pointer_cast(input.data());
+  output_ptr_t d_output = thrust::raw_pointer_cast(output.data());
+
+  input_it_t inp_it(d_input, impl::ChunkToMat<input_t>(p));
+  output_it_t out_it(static_cast<OffsetT>(elements - 1), d_output);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<input_t>(elements, "Sequence Size");
+  state.add_global_memory_writes<accum_t>(1, "Hash Size");
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::InclusiveScan,
+      "InclusiveScan failed",
+      inp_it,
+      out_it, // iterator that only writes the last element of inclusive prefix scan sequence
+      op_t{p},
+      static_cast<OffsetT>(input.size()),
+      env);
+  });
+
+  // for validation uncomment these two lines
+  // assert(impl::validate(input, output, p, bench_stream));
+}
+
+#ifdef TUNE_T
+using type_list = nvbench::type_list<TUNE_T>;
+#else
+// we can split stream of bits into 8-bit, 16-bit, etc. chunks, effectively
+// serving as the number of bits processed by a thread
+using type_list = nvbench::type_list<cuda::std::uint8_t, cuda::std::uint16_t, cuda::std::uint32_t, cuda::std::uint64_t>;
+#endif
+
+NVBENCH_BENCH_TYPES(inclusive_scan, NVBENCH_TYPE_AXES(type_list, offset_types))
+  .set_name("rabin-karp-fingerprinting-monoid")
+  .set_type_axes_names({"BitsetT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("Modulus", {2725841});

cccl_upstream/cub/benchmarks/bench/scan/applications/P1/running-min-max.cu

@@ -0,0 +1,247 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/device_scan.cuh>
+
+#include <thrust/device_vector.h>
+#include <thrust/host_vector.h>
+
+#include <cuda/iterator>
+#include <cuda/std/cmath>
+#include <cuda/std/limits>
+#include <cuda/std/utility>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_DIRECT
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_WARP_TRANSPOSE
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+#endif // !TUNE_BASE
+
+#include "../../policy_selector.h"
+
+namespace impl
+{
+/* Given input sequence of values, compute sequence of
+ * pairs corresponding to running minimum and running maximum values.
+ */
+
+/*! @brief Structure to hold minimum and maximum */
+template <typename T>
+struct min_max_t
+{
+private:
+  T m_min{cuda::std::numeric_limits<T>::max()};
+  T m_max{cuda::std::numeric_limits<T>::min()};
+
+public:
+  min_max_t() = default;
+  __host__ __device__ min_max_t(T minimum, T maximum)
+      : m_min(minimum)
+      , m_max(maximum)
+  {}
+
+  T __host__ __device__ minimum() const
+  {
+    return m_min;
+  }
+  T __host__ __device__ maximum() const
+  {
+    return m_max;
+  }
+};
+
+/* Scan operator combining min-max pairs. It is commutative and associative */
+struct scan_op
+{
+  template <typename T>
+  min_max_t<T> __host__ __device__ operator()(min_max_t<T> v1, min_max_t<T> v2) const
+  {
+    auto min_r = cuda::minimum{}(v1.minimum(), v2.minimum());
+    auto max_r = cuda::maximum{}(v1.maximum(), v2.maximum());
+    return {min_r, max_r};
+  }
+};
+
+template <typename T>
+struct embed_op
+{
+  min_max_t<T> __host__ __device__ operator()(T v) const
+  {
+    return {v, v};
+  }
+};
+
+template <typename T>
+struct extract_min
+{
+  T __host__ __device__ operator()(min_max_t<T> pair) const
+  {
+    return pair.minimum();
+  }
+};
+
+template <typename T>
+struct extract_max
+{
+  T __host__ __device__ operator()(min_max_t<T> pair) const
+  {
+    return pair.maximum();
+  }
+};
+
+template <typename ValueT, typename PairT>
+void validate(const thrust::device_vector<ValueT>& input,
+              const thrust::device_vector<PairT>& output,
+              cudaStream_t stream)
+{
+  using value_t = ValueT;
+  auto elements = input.size();
+
+  thrust::device_vector<value_t> ref_mins(elements, thrust::no_init);
+  thrust::device_vector<value_t> ref_maxs(elements, thrust::no_init);
+
+  size_t tmp_size{};
+  auto d_input  = thrust::raw_pointer_cast(input.data());
+  auto d_output = thrust::raw_pointer_cast(output.data());
+
+  cub::DeviceScan::InclusiveScanInit(
+    nullptr,
+    tmp_size,
+    d_input,
+    ref_mins.begin(),
+    cuda::minimum<>{},
+    cuda::std::numeric_limits<value_t>::max(),
+    input.size(),
+    stream);
+
+  thrust::device_vector<nvbench::uint8_t> tmp1(tmp_size, thrust::no_init);
+  nvbench::uint8_t* d_tmp1 = thrust::raw_pointer_cast(tmp1.data());
+
+  cub::DeviceScan::InclusiveScanInit(
+    d_tmp1,
+    tmp_size,
+    d_input,
+    ref_mins.begin(),
+    cuda::minimum<>{},
+    cuda::std::numeric_limits<value_t>::max(),
+    input.size(),
+    stream);
+
+  cub::DeviceScan::InclusiveScanInit(
+    nullptr,
+    tmp_size,
+    d_input,
+    ref_maxs.begin(),
+    cuda::minimum<>{},
+    cuda::std::numeric_limits<value_t>::max(),
+    input.size(),
+    stream);
+
+  thrust::device_vector<nvbench::uint8_t> tmp2(tmp_size, thrust::no_init);
+  nvbench::uint8_t* d_tmp2 = thrust::raw_pointer_cast(tmp2.data());
+
+  cub::DeviceScan::InclusiveScanInit(
+    d_tmp2,
+    tmp_size,
+    d_input,
+    ref_maxs.begin(),
+    cuda::maximum<>{},
+    cuda::std::numeric_limits<value_t>::min(),
+    input.size(),
+    stream);
+
+  thrust::device_vector<value_t> computed_mins(elements, thrust::no_init);
+  thrust::device_vector<value_t> computed_maxs(elements, thrust::no_init);
+
+  impl::extract_min<value_t> extract_min_op{};
+  cub::DeviceTransform::Transform(d_output, computed_mins.begin(), input.size(), extract_min_op, stream);
+
+  impl::extract_max<value_t> extract_max_op{};
+  cub::DeviceTransform::Transform(d_output, computed_maxs.begin(), input.size(), extract_max_op, stream);
+
+  assert(computed_mins == ref_mins);
+  assert(computed_maxs == ref_maxs);
+}
+}; // namespace impl
+
+template <typename T, typename OffsetT>
+void benchmark_impl(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using value_t                  = T;
+  using pair_t                   = impl::min_max_t<value_t>;
+  using op_t                     = impl::scan_op;
+  using accum_t [[maybe_unused]] = pair_t;
+  using input_raw_t              = const value_t*;
+  using input_it_t               = cuda::transform_iterator<impl::embed_op<value_t>, input_raw_t>;
+  using output_it_t              = pair_t*;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<pair_t> output(elements);
+  thrust::device_vector<value_t> input = generate(elements);
+
+  input_raw_t d_input  = thrust::raw_pointer_cast(input.data());
+  output_it_t d_output = thrust::raw_pointer_cast(output.data());
+
+  input_it_t inp_it(d_input, impl::embed_op<value_t>{});
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<value_t>(elements, "Size");
+  state.add_global_memory_writes<pair_t>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::InclusiveScan,
+      "InclusiveScan failed",
+      inp_it,
+      d_output,
+      op_t{},
+      static_cast<OffsetT>(input.size()),
+      env);
+  });
+
+  // for verification use
+  // impl::validate(input, output, state.get_cuda_stream().get_stream());
+}
+
+#ifdef TUNE_T
+using bench_types = nvbench::type_list<TUNE_T>;
+#else
+using bench_types = nvbench::type_list<nvbench::uint32_t, nvbench::int64_t, nvbench::float32_t, nvbench::float64_t>;
+#endif
+
+NVBENCH_BENCH_TYPES(benchmark_impl, NVBENCH_TYPE_AXES(bench_types, offset_types))
+  .set_name("running-min-max")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/scan/applications/P1/scan-over-unitriangular-group.cu

@@ -0,0 +1,165 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/device_scan.cuh>
+
+#include <thrust/device_vector.h>
+#include <thrust/host_vector.h>
+
+#include <cuda/iterator>
+#include <cuda/std/tuple>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_DIRECT
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_WARP_TRANSPOSE
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+#endif // !TUNE_BASE
+
+#include "../../policy_selector.h"
+
+namespace impl
+{
+template <typename T>
+using triplet_t = cuda::std::tuple<T, T, T>;
+
+/* The triplet corresponds to strictly upper triangular elements of a unitriangular matrix
+   A = [[1, a1, a12], [0, 1, a2], [0, 0, 1]], mapped to triplet [a1, a2, a12].
+
+   The set of unitriangular matrix forms a group, with product induced by matrix multiplication,
+   and the identity element corresponding to zero triplet.
+*/
+struct unitriangular_dim3_op
+{
+  // Scan operation: associative and non-commutative
+  template <typename T>
+  triplet_t<T> __host__ __device__ operator()(triplet_t<T> a, triplet_t<T> b) const
+  {
+    auto [a1, a2, a12] = a;
+    auto [b1, b2, b12] = b;
+
+    return {a1 + b1, a2 + b1, a12 + b12 + a1 * b2};
+  }
+};
+
+// Utility operation to pack arguments into a triplet_t instance
+struct pack_op
+{
+  template <typename T>
+  triplet_t<T> __host__ __device__ operator()(T a1, T a2, T a12) const
+  {
+    return {a1, a2, a12};
+  } // namespace impl
+};
+
+template <typename TupleT, typename ScanOpT>
+bool validation(const thrust::device_vector<TupleT>& input,
+                const thrust::device_vector<TupleT>& output,
+                ScanOpT op,
+                cudaStream_t stream)
+{
+  cudaStreamSynchronize(stream);
+
+  using tuple_t = TupleT;
+  thrust::host_vector<tuple_t> h_input(input);
+  thrust::host_vector<tuple_t> h_output(output);
+
+  auto elements = input.size();
+  thrust::host_vector<tuple_t> h_reference(elements);
+
+  h_reference[0] = h_input[0];
+  for (std::size_t i = 1; i < elements; ++i)
+  {
+    h_reference[i] = op(h_reference[i - 1], h_input[i]);
+  }
+
+  return h_reference == h_output;
+}
+}; // namespace impl
+
+template <typename T, typename OffsetT>
+void benchmark_impl(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using value_t                  = T;
+  using tuple_t                  = impl::triplet_t<value_t>;
+  using op_t                     = impl::unitriangular_dim3_op;
+  using accum_t [[maybe_unused]] = tuple_t;
+
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  cudaStream_t bench_stream = state.get_cuda_stream().get_stream();
+
+  thrust::device_vector<tuple_t> output(elements);
+  thrust::device_vector<value_t> _input = generate(cuda::std::tuple_size_v<tuple_t> * elements);
+  thrust::device_vector<tuple_t> input(elements);
+
+  cub::DeviceTransform::Transform(
+    cuda::std::make_tuple(cuda::strided_iterator(_input.begin(), std::size_t{3}),
+                          cuda::strided_iterator(_input.begin() + 1, std::size_t{3}),
+                          cuda::strided_iterator(_input.begin() + 2, std::size_t{3})),
+    input.begin(),
+    input.size(),
+    impl::pack_op{},
+    bench_stream);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<tuple_t>(elements, "Size");
+  state.add_global_memory_writes<tuple_t>(elements);
+
+  auto d_input  = thrust::raw_pointer_cast(input.data());
+  auto d_output = thrust::raw_pointer_cast(output.data());
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::InclusiveScan,
+      "InclusiveScan failed",
+      d_input,
+      d_output,
+      op_t{},
+      static_cast<OffsetT>(input.size()),
+      env);
+  });
+
+  // for validation use (recommended for integral types and smallish input sizes)
+  // assert(impl::validation(input, output, op_t{}, bench_stream));
+}
+
+#ifdef TUNE_T
+using bench_types = nvbench::type_list<TUNE_T>;
+#else
+using bench_types = nvbench::type_list<nvbench::int32_t, nvbench::uint64_t, nvbench::float32_t, nvbench::float64_t>;
+#endif
+
+NVBENCH_BENCH_TYPES(benchmark_impl, NVBENCH_TYPE_AXES(bench_types, offset_types))
+  .set_name("unitriangular-monoid")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/scan/exclusive/base.cuh

@@ -0,0 +1,88 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#pragma once
+
+#include <cub/device/device_scan.cuh>
+
+#include <cuda/std/__functional/invoke.h>
+
+#include <nvbench_helper.cuh>
+
+#include "../policy_selector.h"
+
+template <typename T, typename OffsetT>
+static void basic(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+try
+{
+  using init_value_t             = T;
+  using accum_t [[maybe_unused]] = ::cuda::std::__accumulator_t<op_t, init_value_t, T>;
+  using offset_t                 = cub::detail::choose_offset_t<OffsetT>;
+#if USES_LOOKAHEAD()
+  static_assert(sizeof(offset_t) == sizeof(size_t)); // lookahead scan uses size_t internally
+#endif // USES_LOOKAHEAD()
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  if (sizeof(offset_t) == 4 && elements > std::numeric_limits<offset_t>::max())
+  {
+    state.skip("Skipping: input size exceeds 32-bit offset type capacity.");
+    return;
+  }
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements);
+
+  const T* d_input = thrust::raw_pointer_cast(input.data());
+  T* d_output      = thrust::raw_pointer_cast(output.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::ExclusiveScan,
+      "ExclusiveScan failed",
+      d_input,
+      d_output,
+      op_t{},
+      init_value_t{},
+      static_cast<offset_t>(input.size()),
+      env);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+// __half and __nv_bfloat16 are added for full (non-tuning) runs; CUB has fast paths for them (see #9587).
+#ifdef TUNE_T
+using value_types = nvbench::type_list<TUNE_T>;
+#else
+using value_types =
+  push_back_t<all_types
+#  if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
+              ,
+              __half
+#  endif
+#  if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
+              ,
+              __nv_bfloat16
+#  endif
+              >;
+#endif
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(value_types, scan_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/scan/exclusive/by_key.cu

@@ -0,0 +1,106 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_scan.cuh>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#if !TUNE_BASE
+struct bench_scan_by_key_policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ScanByKeyPolicy
+  {
+    return {cub::ScanByKeyAlgorithm::lookback,
+            {TUNE_THREADS,
+             TUNE_ITEMS,
+             TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE,
+             TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : cub::LOAD_CA,
+             TUNE_TRANSPOSE == 0 ? cub::BLOCK_STORE_DIRECT : cub::BLOCK_STORE_WARP_TRANSPOSE,
+             cub::BLOCK_SCAN_WARP_SCANS,
+             lookback_delay_policy}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename KeyT, typename ValueT, typename OffsetT>
+static void scan(nvbench::state& state, nvbench::type_list<KeyT, ValueT, OffsetT>)
+{
+  using init_value_t  = ValueT;
+  using op_t          = ::cuda::std::plus<>;
+  using equality_op_t = ::cuda::std::equal_to<>;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<ValueT> in_vals(elements);
+  thrust::device_vector<ValueT> out_vals(elements);
+  thrust::device_vector<KeyT> keys = generate.uniform.key_segments(elements, 0, 5200);
+
+  const KeyT* d_keys      = thrust::raw_pointer_cast(keys.data());
+  const ValueT* d_in_vals = thrust::raw_pointer_cast(in_vals.data());
+  ValueT* d_out_vals      = thrust::raw_pointer_cast(out_vals.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<ValueT>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_scan_by_key_policy_selector{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::ExclusiveScanByKey,
+      "ExclusiveScanByKey failed",
+      d_keys,
+      d_in_vals,
+      d_out_vals,
+      op_t{},
+      init_value_t{},
+      static_cast<OffsetT>(elements),
+      equality_op_t{},
+      env);
+  });
+}
+
+using some_offset_types = nvbench::type_list<nvbench::int32_t>;
+
+#ifdef TUNE_KeyT
+using key_types = nvbench::type_list<TUNE_KeyT>;
+#else // !defined(TUNE_KeyT)
+using key_types = all_types;
+#endif // TUNE_KeyT
+
+#ifdef TUNE_ValueT
+using value_types = nvbench::type_list<TUNE_ValueT>;
+#else // !defined(TUNE_ValueT)
+using value_types =
+  nvbench::type_list<int8_t,
+                     int16_t,
+                     int32_t,
+                     int64_t
+#  if _CCCL_HAS_INT128()
+                     ,
+                     int128_t
+#  endif
+                     >;
+#endif // TUNE_ValueT
+
+NVBENCH_BENCH_TYPES(scan, NVBENCH_TYPE_AXES(key_types, value_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/scan/exclusive/custom.cu

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+// This benchmark uses a custom operation, max_t, which is not known to CUB, so no operator specific optimizations and
+// tunings are performed.
+
+// Because CUB cannot detect this operator, we cannot add any tunings based on the results of this benchmark. Its main
+// use is to detect regressions.
+
+#include <nvbench_helper.cuh>
+
+#define USES_LOOKAHEAD() 0
+using op_t              = max_t;
+using scan_offset_types = offset_types;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/scan/exclusive/deterministic.cu

@@ -0,0 +1,57 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/device/device_scan.cuh>
+
+#include <cuda/__execution/determinism.h>
+#include <cuda/__execution/require.h>
+#include <cuda/std/__functional/invoke.h>
+
+#include <nvbench_helper.cuh>
+
+template <typename T, typename OffsetT>
+static void exclusive_scan(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+try
+{
+  using init_value_t = T;
+  using offset_t     = OffsetT;
+  using scan_op_t    = ::cuda::std::plus<T>;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements, thrust::no_init);
+
+  const T* d_input = thrust::raw_pointer_cast(input.data());
+  T* d_output      = thrust::raw_pointer_cast(output.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(alloc, launch, cuda::execution::require(cuda::execution::determinism::run_to_run));
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::ExclusiveScan,
+      "ExclusiveScan failed",
+      d_input,
+      d_output,
+      scan_op_t{},
+      init_value_t{},
+      static_cast<offset_t>(elements),
+      env);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+using types   = nvbench::type_list<float, double>;
+using offsets = nvbench::type_list<int64_t>;
+
+NVBENCH_BENCH_TYPES(exclusive_scan, NVBENCH_TYPE_AXES(types, offsets))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/scan/exclusive/sum.cu

@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+// Tuning parameters found for signed integer types apply equally for unsigned integer types
+
+#include <nvbench_helper.cuh>
+
+// This benchmark tunes the old, non-lookahead scan implementation. Using it for benchmarking, will pick the lookahead
+// implementation on SM100+, but it's better to use the sum.lookahead.cu benchmark instead, which uses a single OffsetT.
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#define USES_LOOKAHEAD() 0
+using op_t              = ::cuda::std::plus<>;
+using scan_offset_types = offset_types;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/scan/exclusive/sum.lookahead.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// This tunes the lookahead implementation of scan, which is only available on SM100+. It has entirely different tuning
+// parameters and is agnostic of the offset type. It is thus in a separate file, so we can continue to tune the old scan
+// implementation on older hardware architectures.
+
+#include <cuda/__cccl_config>
+
+#if _CCCL_PP_COUNT(__CUDA_ARCH_LIST__) != 1
+#  warning "This benchmark does not support being compiled for multiple architectures. Disabling it."
+#else // _CCCL_PP_COUNT(__CUDA_ARCH_LIST__) != 1
+
+#  if __CUDA_ARCH_LIST__ < 1000
+// We don't care if clang-tidy can't parse this
+#    ifndef _CCCL_CLANG_TIDY_INVOKED
+#      warning "Lookahead scan requires at least sm_100. Disabling it."
+#    endif // !defined _CCCL_CLANG_TIDY_INVOKED
+#  else // __CUDA_ARCH_LIST__ < 1000
+
+#    if __cccl_ptx_isa < 860
+#      warning "Lookahead scan requires at least PTX ISA 8.6. Disabling it."
+#    else // if __cccl_ptx_isa < 860
+
+#      include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_NUM_REDUCE_SCAN_WARPS wrps 1:8:1
+// %RANGE% TUNE_NUM_LOOKBACK_ITEMS lbi 1:8:1
+
+// TODO(bgruber): find a good range and step width, items per thread should be coprime with 32 to avoid SMEM conflicts.
+// Should we specify nominal items per thread instead?
+// %RANGE% TUNE_ITEMS_PLUS_ONE ipt 8:256:8
+
+// %RANGE% TUNE_LOOKBACK_STAGES lbs -2:2:1
+// %RANGE% TUNE_BLOCK_IDX_STAGES bis -2:2:1
+
+#      define USES_LOOKAHEAD() 1
+using op_t              = ::cuda::std::plus<>;
+using scan_offset_types = nvbench::type_list<int64_t>;
+#      include "base.cuh"
+
+#    endif // __cccl_ptx_isa < 860
+#  endif // __CUDA_ARCH_LIST__ < 1000
+#endif // _CCCL_PP_COUNT(__CUDA_ARCH_LIST__) != 1

cccl_upstream/cub/benchmarks/bench/scan/policy_selector.h

@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/device/device_scan.cuh>
+
+#ifndef USES_LOOKAHEAD
+#  define USES_LOOKAHEAD() 0
+#endif
+
+#if !TUNE_BASE
+#  if !USES_LOOKAHEAD()
+#    include <look_back_helper.cuh>
+#  endif // !USES_LOOKAHEAD()
+
+template <typename AccumT>
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ScanPolicy
+  {
+#  if USES_LOOKAHEAD()
+    return {cub::ScanAlgorithm::lookahead,
+            cub::ScanLookbackPolicy{},
+            cub::ScanLookaheadPolicy{
+              TUNE_NUM_REDUCE_SCAN_WARPS,
+              TUNE_ITEMS_PLUS_ONE - 1,
+              TUNE_NUM_LOOKBACK_ITEMS,
+              TUNE_LOOKBACK_STAGES,
+              TUNE_BLOCK_IDX_STAGES}};
+#  else
+    return cub::detail::scan::make_mem_scaled_lookback_scan_policy(
+      TUNE_THREADS,
+      TUNE_ITEMS,
+      int{sizeof(AccumT)},
+      TUNE_LOAD_ALGORITHM,
+      TUNE_LOAD_MODIFIER,
+      TUNE_STORE_ALGORITHM,
+      cub::BLOCK_SCAN_WARP_SCANS,
+      lookback_delay_policy);
+#  endif
+  }
+};
+#endif // !TUNE_BASE