[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/cub/benchmarks/CMakeLists.txt

@@ -0,0 +1,160 @@
+include(${CMAKE_SOURCE_DIR}/benchmarks/cmake/CCCLBenchmarkRegistry.cmake)
+
+cccl_get_nvbench_helper()
+
+set(benches_root "${CMAKE_CURRENT_LIST_DIR}")
+
+if (NOT CMAKE_BUILD_TYPE STREQUAL "Release")
+  set(message_type FATAL_ERROR)
+  if (CCCL_ENABLE_CLANG_TIDY)
+    # We are here because CI has force-enabled clang-tidy. We must use a debug build for
+    # this because certain clang-tidy checks (such as out of bounds or clang static
+    # analyzer) work better when they see assert()'s. In this case we don't actually
+    # intend to run any of the benchmarks, we just need them to be compilable, so a simple
+    # warning is enough.
+    #
+    # We don't ignore this outright (by making it say, DEBUG or VERBOSE), because it's
+    # possible that a user may accidentally stumble into enabling the option.
+    set(message_type WARNING)
+  endif()
+  message(${message_type} "CUB benchmarks must be built in release mode.")
+endif()
+
+if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+  message(
+    FATAL_ERROR
+    "CMAKE_CUDA_ARCHITECTURES must be set to build CUB benchmarks."
+  )
+endif()
+
+set(benches_meta_target cub.all.benches)
+add_custom_target(${benches_meta_target})
+
+function(get_recursive_subdirs subdirs)
+  set(dirs)
+  file(
+    GLOB_RECURSE contents
+    CONFIGURE_DEPENDS
+    LIST_DIRECTORIES ON
+    "${CMAKE_CURRENT_LIST_DIR}/bench/*"
+  )
+
+  foreach (test_dir IN LISTS contents)
+    if (IS_DIRECTORY "${test_dir}")
+      list(APPEND dirs "${test_dir}")
+    endif()
+  endforeach()
+
+  set(${subdirs} "${dirs}" PARENT_SCOPE)
+endfunction()
+
+create_benchmark_registry()
+
+function(get_bench_ranges src bench_name)
+  file(READ "${src}" file_data)
+  set(param_regex "//[ ]+%RANGE%[ ]+([^ ]+)[ ]+([^ ]+)[ ]+([^\n]*)")
+
+  string(REGEX MATCHALL "${param_regex}" matches "${file_data}")
+
+  set(ranges "")
+
+  foreach (match IN LISTS matches)
+    string(REGEX MATCH "${param_regex}" unused "${match}")
+
+    set(def ${CMAKE_MATCH_1})
+    set(label ${CMAKE_MATCH_2})
+    set(range ${CMAKE_MATCH_3})
+    set(ranges "${ranges}${def}|${label}=${range},")
+
+    string(REPLACE ":" ";" range "${range}")
+    list(LENGTH range range_len)
+
+    if (NOT "${range_len}" STREQUAL 3)
+      message(FATAL_ERROR "Range should be represented as 'start:end:step'")
+    endif()
+  endforeach()
+
+  string(LENGTH "${ranges}" ranges_length)
+  math(EXPR last_character_index "${ranges_length} - 1")
+  string(SUBSTRING "${ranges}" 0 ${last_character_index} ranges)
+  register_cccl_tuning("${bench_name}" "${ranges}")
+endfunction()
+
+function(add_bench target_name bench_name bench_src)
+  set(bench_target ${bench_name})
+  set(${target_name} ${bench_target} PARENT_SCOPE)
+
+  cccl_add_executable(${bench_target} SOURCES "${bench_src}")
+  target_link_libraries(
+    ${bench_target}
+    PRIVATE #
+      cccl.nvbench_helper
+      nvbench::main
+  )
+
+  if (
+    CCCL_ENABLE_EXPERIMENTAL_TILE_TRANSFORM_DISPATCH
+    AND "${bench_src}" MATCHES "/transform/tile/"
+  )
+    target_compile_options(
+      ${bench_target}
+      PRIVATE "$<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--enable-tile>"
+    )
+    target_compile_definitions(
+      ${bench_target}
+      PRIVATE _CCCL_ENABLE_EXPERIMENTAL_TILE_TRANSFORM_DISPATCH
+    )
+  endif()
+endfunction()
+
+function(add_bench_dir bench_dir)
+  file(GLOB bench_srcs CONFIGURE_DEPENDS "${bench_dir}/*.cu")
+  file(RELATIVE_PATH bench_prefix "${benches_root}" "${bench_dir}")
+  file(TO_CMAKE_PATH "${bench_prefix}" bench_prefix)
+  string(REPLACE "/" "." bench_prefix "${bench_prefix}")
+
+  foreach (bench_src IN LISTS bench_srcs)
+    # base tuning
+    get_filename_component(bench_name "${bench_src}" NAME_WLE)
+    string(PREPEND bench_name "cub.${bench_prefix}.")
+
+    set(base_bench_name "${bench_name}.base")
+    add_bench(base_bench_target ${base_bench_name} "${bench_src}")
+    add_dependencies(${benches_meta_target} ${base_bench_target})
+    target_compile_definitions(${base_bench_target} PRIVATE TUNE_BASE=1)
+    target_compile_options(
+      ${base_bench_target}
+      PRIVATE "$<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--extended-lambda>"
+    )
+
+    if (CUB_ENABLE_TUNING)
+      # tuning
+      set_property(
+        DIRECTORY
+        APPEND
+        PROPERTY CMAKE_CONFIGURE_DEPENDS "${bench_src}"
+      )
+      get_bench_ranges("${bench_src}" "${bench_name}")
+      set(tuning_name "${bench_name}.variant")
+      set(tuning_path "${CMAKE_BINARY_DIR}/${tuning_name}.h")
+      add_bench(bench_target ${tuning_name} "${bench_src}")
+      # for convenience, make tuning variant buildable by default
+      file(WRITE "${tuning_path}" "#pragma once\n#define TUNE_BASE 1\n")
+      target_compile_options(
+        ${bench_target}
+        PRIVATE #
+          "$<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--extended-lambda>"
+          "-include${tuning_path}"
+      )
+    else()
+      # benchmarking
+      register_cccl_benchmark("${bench_name}" "")
+    endif()
+  endforeach()
+endfunction()
+
+get_recursive_subdirs(subdirs)
+
+foreach (subdir IN LISTS subdirs)
+  add_bench_dir("${subdir}")
+endforeach()

cccl_upstream/cub/benchmarks/bench/adjacent_difference/subtract_left.cu

@@ -0,0 +1,71 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_adjacent_difference.cuh>
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+
+#if !TUNE_BASE
+struct policy_selector_t
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const
+    -> cub::AdjacentDifferencePolicy
+  {
+    return {TUNE_THREADS_PER_BLOCK,
+            TUNE_ITEMS_PER_THREAD,
+            cub::BLOCK_LOAD_WARP_TRANSPOSE,
+            cub::LOAD_CA,
+            cub::BLOCK_STORE_WARP_TRANSPOSE};
+  }
+};
+#endif // !TUNE_BASE
+
+template <class T, class OffsetT>
+void left(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using input_it_t      = const T*;
+  using output_it_t     = T*;
+  using difference_op_t = ::cuda::std::minus<>;
+  using offset_t        = cub::detail::choose_offset_t<OffsetT>;
+
+  const auto elements         = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<T> out(elements, thrust::no_init);
+
+  input_it_t d_in   = thrust::raw_pointer_cast(in.data());
+  output_it_t d_out = thrust::raw_pointer_cast(out.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector_t{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceAdjacentDifference::SubtractLeftCopy,
+      "SubtractLeftCopy failed",
+      d_in,
+      d_out,
+      static_cast<offset_t>(elements),
+      difference_op_t{},
+      env);
+  });
+}
+
+using types = nvbench::type_list<int32_t>;
+
+NVBENCH_BENCH_TYPES(left, NVBENCH_TYPE_AXES(types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/copy/memcpy.cu

@@ -0,0 +1,252 @@
+// SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_memcpy.cuh>
+
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_BUFFERS_PER_THREAD bpt 1:18:1
+// %RANGE% TUNE_TLEV_BYTES_PER_THREAD tlevbpt 2:16:2
+// %RANGE% TUNE_LARGE_THREADS ltpb 128:1024:32
+// %RANGE% TUNE_LARGE_BUFFER_BYTES_PER_THREAD lbbpt 4:128:4
+// %RANGE% TUNE_PREFER_POW2_BITS ppb 0:1:1
+// %RANGE% TUNE_WARP_LEVEL_THRESHOLD wlt 32:512:32
+// %RANGE% TUNE_BLOCK_LEVEL_THRESHOLD blt 1024:16384:512
+// %RANGE% TUNE_BLOCK_MAGIC_NS blns 0:2048:4
+// %RANGE% TUNE_BLOCK_DELAY_CONSTRUCTOR_ID bldcid 0:7:1
+// %RANGE% TUNE_BLOCK_L2_WRITE_LATENCY_NS bll2w 0:1200:5
+// %RANGE% TUNE_BUFF_MAGIC_NS buns 0:2048:4
+// %RANGE% TUNE_BUFF_DELAY_CONSTRUCTOR_ID budcid 0:7:1
+// %RANGE% TUNE_BUFF_L2_WRITE_LATENCY_NS bul2w 0:1200:5
+
+#include <thrust/random.h>
+#include <thrust/scan.h>
+#include <thrust/scatter.h>
+#include <thrust/sequence.h>
+#include <thrust/shuffle.h>
+#include <thrust/tabulate.h>
+
+#include <nvbench_helper.cuh>
+
+template <class T, class OffsetT>
+struct offset_to_ptr_t
+{
+  T* d_ptr;
+  OffsetT* d_offsets;
+
+  __device__ T* operator()(OffsetT i) const
+  {
+    return d_ptr + d_offsets[i];
+  }
+};
+
+template <class T, class OffsetT>
+struct reordered_offset_to_ptr_t
+{
+  T* d_ptr;
+  OffsetT* d_map;
+  OffsetT* d_offsets;
+
+  __device__ T* operator()(OffsetT i) const
+  {
+    return d_ptr + d_offsets[d_map[i]];
+  }
+};
+
+template <class T, class OffsetT>
+struct offset_to_bytes_t
+{
+  OffsetT* d_offsets;
+
+  __device__ OffsetT operator()(OffsetT i) const
+  {
+    return (d_offsets[i + 1] - d_offsets[i]) * sizeof(T);
+  }
+};
+
+template <class T, class OffsetT>
+struct offset_to_size_t
+{
+  OffsetT* d_offsets;
+
+  __device__ OffsetT operator()(OffsetT i) const
+  {
+    return d_offsets[i + 1] - d_offsets[i];
+  }
+};
+
+#if !TUNE_BASE
+
+struct policy_selector_t
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::BatchedCopyPolicy
+  {
+    return {
+      cub::BatchedCopyAlgorithm::lookback,
+      {
+        {
+          TUNE_THREADS,
+          TUNE_BUFFERS_PER_THREAD,
+          TUNE_TLEV_BYTES_PER_THREAD,
+          bool{TUNE_PREFER_POW2_BITS},
+          TUNE_LARGE_THREADS * TUNE_LARGE_BUFFER_BYTES_PER_THREAD,
+          TUNE_WARP_LEVEL_THRESHOLD,
+          TUNE_BLOCK_LEVEL_THRESHOLD,
+          cub::LookbackDelayPolicy{static_cast<cub::LookbackDelayAlgorithm>(TUNE_BUFF_DELAY_CONSTRUCTOR_ID),
+                                   TUNE_BUFF_MAGIC_NS,
+                                   TUNE_BUFF_L2_WRITE_LATENCY_NS},
+          cub::LookbackDelayPolicy{static_cast<cub::LookbackDelayAlgorithm>(TUNE_BLOCK_DELAY_CONSTRUCTOR_ID),
+                                   TUNE_BLOCK_MAGIC_NS,
+                                   TUNE_BLOCK_L2_WRITE_LATENCY_NS},
+
+        },
+        {TUNE_LARGE_THREADS, TUNE_LARGE_BUFFER_BYTES_PER_THREAD},
+      },
+    };
+  }
+};
+#endif
+
+template <class T, class OffsetT>
+void gen_it(T* d_buffer,
+            thrust::device_vector<T*>& output,
+            thrust::device_vector<OffsetT> offsets,
+            bool randomize,
+            thrust::default_random_engine& rne)
+{
+  OffsetT* d_offsets = thrust::raw_pointer_cast(offsets.data());
+
+  if (randomize)
+  {
+    const auto buffers = output.size();
+    thrust::device_vector<OffsetT> map(buffers);
+    thrust::sequence(map.begin(), map.end());
+    thrust::shuffle(map.begin(), map.end(), rne);
+    thrust::device_vector<OffsetT> sizes(buffers);
+    thrust::tabulate(sizes.begin(), sizes.end(), offset_to_size_t<T, OffsetT>{d_offsets});
+    thrust::scatter(sizes.begin(), sizes.end(), map.begin(), offsets.begin());
+    thrust::exclusive_scan(offsets.begin(), offsets.end(), offsets.begin());
+    OffsetT* d_map = thrust::raw_pointer_cast(map.data());
+    thrust::tabulate(output.begin(), output.end(), reordered_offset_to_ptr_t<T, OffsetT>{d_buffer, d_map, d_offsets});
+  }
+  else
+  {
+    thrust::tabulate(output.begin(), output.end(), offset_to_ptr_t<T, OffsetT>{d_buffer, d_offsets});
+  }
+}
+
+template <class T, class OffsetT>
+void copy(nvbench::state& state,
+          nvbench::type_list<T, OffsetT>,
+          std::size_t elements,
+          std::size_t min_buffer_size,
+          std::size_t max_buffer_size,
+          bool randomize_input,
+          bool randomize_output)
+{
+  using offset_t           = OffsetT;
+  using it_t               = T*;
+  using input_buffer_it_t  = it_t*;
+  using output_buffer_it_t = it_t*;
+  using buffer_size_it_t   = offset_t*;
+
+  thrust::device_vector<T> input_buffer = generate(elements);
+  thrust::device_vector<T> output_buffer(elements);
+  thrust::device_vector<offset_t> offsets =
+    generate.uniform.segment_offsets(elements, min_buffer_size, max_buffer_size);
+
+  T* d_input_buffer   = thrust::raw_pointer_cast(input_buffer.data());
+  T* d_output_buffer  = thrust::raw_pointer_cast(output_buffer.data());
+  offset_t* d_offsets = thrust::raw_pointer_cast(offsets.data());
+
+  const auto buffers = offsets.size() - 1;
+
+  thrust::device_vector<it_t> input_buffers(buffers);
+  thrust::device_vector<it_t> output_buffers(buffers);
+  thrust::device_vector<offset_t> buffer_sizes(buffers);
+  thrust::tabulate(buffer_sizes.begin(), buffer_sizes.end(), offset_to_bytes_t<T, offset_t>{d_offsets});
+
+  thrust::default_random_engine rne;
+  gen_it(d_input_buffer, input_buffers, offsets, randomize_input, rne);
+  gen_it(d_output_buffer, output_buffers, offsets, randomize_output, rne);
+
+  // Clear the offsets vector to free memory
+  offsets.clear();
+  offsets.shrink_to_fit();
+  d_offsets = nullptr;
+
+  input_buffer_it_t d_input_buffers   = thrust::raw_pointer_cast(input_buffers.data());
+  output_buffer_it_t d_output_buffers = thrust::raw_pointer_cast(output_buffers.data());
+  buffer_size_it_t d_buffer_sizes     = thrust::raw_pointer_cast(buffer_sizes.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_writes<T>(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_reads<it_t>(buffers);
+  state.add_global_memory_reads<it_t>(buffers);
+  state.add_global_memory_reads<offset_t>(buffers);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               auto env = cub_bench_env(
+                 alloc,
+                 launch
+#if !TUNE_BASE
+                 ,
+                 cuda::execution::tune(policy_selector_t{})
+#endif
+               );
+               _CCCL_TRY_CUDA_API(
+                 cub::DeviceMemcpy::Batched,
+                 "Batched failed",
+                 d_input_buffers,
+                 d_output_buffers,
+                 d_buffer_sizes,
+                 static_cast<cuda::std::int64_t>(buffers),
+                 env);
+             });
+}
+
+template <class T, class OffsetT>
+void uniform(nvbench::state& state, nvbench::type_list<T, OffsetT> tl)
+{
+  const auto elements              = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const auto max_buffer_size       = static_cast<std::size_t>(state.get_int64("MaxBufferSize"));
+  const auto min_buffer_size_ratio = static_cast<std::size_t>(state.get_int64("MinBufferSizeRatio"));
+  const auto min_buffer_size =
+    static_cast<std::size_t>(static_cast<double>(max_buffer_size) / 100.0) * min_buffer_size_ratio;
+
+  copy(
+    state, tl, elements, min_buffer_size, max_buffer_size, state.get_int64("Randomize"), state.get_int64("Randomize"));
+}
+
+template <class T, class OffsetT>
+void large(nvbench::state& state, nvbench::type_list<T, OffsetT> tl)
+{
+  const auto elements                  = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const auto max_buffer_size           = elements;
+  constexpr auto min_buffer_size_ratio = 99;
+  const auto min_buffer_size =
+    static_cast<std::size_t>(static_cast<double>(max_buffer_size) / 100.0) * min_buffer_size_ratio;
+
+  // No need to randomize large buffers
+  constexpr bool randomize_input  = false;
+  constexpr bool randomize_output = false;
+
+  copy(state, tl, elements, min_buffer_size, max_buffer_size, randomize_input, randomize_output);
+}
+
+using types = nvbench::type_list<nvbench::uint8_t, nvbench::uint32_t>;
+
+NVBENCH_BENCH_TYPES(uniform, NVBENCH_TYPE_AXES(types, offset_types))
+  .set_name("uniform")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(25, 29, 2))
+  .add_int64_axis("MinBufferSizeRatio", {1, 99})
+  .add_int64_axis("MaxBufferSize", {8, 64, 256, 1024, 64 * 1024})
+  .add_int64_axis("Randomize", {0, 1});
+
+NVBENCH_BENCH_TYPES(large, NVBENCH_TYPE_AXES(types, offset_types))
+  .set_name("large")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", {28, 29});

cccl_upstream/cub/benchmarks/bench/find_bound/find_bound_common.cuh

@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#pragma once
+
+//! Shared setup for `cub::DeviceFind` bounds benchmarks. Data layout and
+//! generation mirror `thrust/benchmarks/bench/vectorized_search/{lower,upper}_bound.cu`
+//! (Elements pow2 16..28 step 4, int8..int64, NeedlesRatio {1, 25, 50}).
+
+#include <thrust/device_vector.h>
+#include <thrust/sort.h>
+
+#include <cstddef>
+
+#include <nvbench_helper.cuh>
+
+template <typename T>
+struct bounds_bench_data
+{
+  thrust::device_vector<T> data{};
+  thrust::device_vector<std::ptrdiff_t> result{};
+  std::size_t elements{};
+  std::size_t needles{};
+
+  explicit bounds_bench_data(nvbench::state& state)
+  {
+    elements                 = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+    const auto needles_ratio = static_cast<std::size_t>(state.get_int64("NeedlesRatio"));
+    needles                  = needles_ratio * static_cast<std::size_t>(static_cast<double>(elements) / 100.0);
+
+    data   = generate(elements + needles);
+    result = thrust::device_vector<std::ptrdiff_t>(needles, thrust::no_init);
+
+    thrust::sort(data.begin(),
+                 data.begin() + static_cast<typename thrust::device_vector<T>::difference_type>(elements));
+  }
+
+  void sort_needles()
+  {
+    thrust::sort(data.begin() + static_cast<typename thrust::device_vector<T>::difference_type>(elements), data.end());
+  }
+
+  T* range_ptr()
+  {
+    return thrust::raw_pointer_cast(data.data());
+  }
+
+  T* values_ptr()
+  {
+    return thrust::raw_pointer_cast(
+      data.data() + static_cast<typename thrust::device_vector<T>::difference_type>(elements));
+  }
+
+  std::ptrdiff_t* output_ptr()
+  {
+    return thrust::raw_pointer_cast(result.data());
+  }
+};

cccl_upstream/cub/benchmarks/bench/find_bound/lower_bound.cu

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+//! `cub::DeviceFind::LowerBound`: haystack sorted, needles unsorted (Thrust vectorized_search parity).
+
+#include <cub/device/device_find.cuh>
+
+#include <cstdint>
+
+#include <nvbench_helper.cuh>
+
+#include "find_bound_common.cuh"
+
+template <typename T>
+void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  bounds_bench_data<T> s(state);
+
+  state.add_element_count(s.needles);
+  state.add_global_memory_reads<T>(s.elements + s.needles);
+  state.add_global_memory_writes<std::ptrdiff_t>(s.needles);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    const auto env = cub_bench_env(alloc, launch);
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceFind::LowerBound,
+      "LowerBound failed",
+      s.range_ptr(),
+      static_cast<std::int64_t>(s.elements),
+      s.values_ptr(),
+      static_cast<std::int64_t>(s.needles),
+      s.output_ptr(),
+      less_t{},
+      env);
+  });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(integral_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("NeedlesRatio", {1, 25, 50});

cccl_upstream/cub/benchmarks/bench/find_bound/lower_bound_sorted_values.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+//! `cub::DeviceFind::LowerBoundSortedValues`: haystack and values (needles) sorted.
+
+#include <cub/device/device_find.cuh>
+
+#include <cstdint>
+
+#include <nvbench_helper.cuh>
+
+#include "find_bound_common.cuh"
+
+template <typename T>
+void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  bounds_bench_data<T> s(state);
+  s.sort_needles();
+
+  state.add_element_count(s.needles);
+  state.add_global_memory_reads<T>(s.elements + s.needles);
+  state.add_global_memory_writes<std::ptrdiff_t>(s.needles);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    const auto env = cub_bench_env(alloc, launch);
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceFind::LowerBoundSortedValues,
+      "LowerBoundSortedValues failed",
+      s.range_ptr(),
+      static_cast<std::int64_t>(s.elements),
+      s.values_ptr(),
+      static_cast<std::int64_t>(s.needles),
+      s.output_ptr(),
+      less_t{},
+      env);
+  });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(integral_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("NeedlesRatio", {1, 25, 50});

cccl_upstream/cub/benchmarks/bench/find_bound/upper_bound.cu

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+//! `cub::DeviceFind::UpperBound`: haystack sorted, needles unsorted (Thrust vectorized_search parity).
+
+#include <cub/device/device_find.cuh>
+
+#include <cstdint>
+
+#include <nvbench_helper.cuh>
+
+#include "find_bound_common.cuh"
+
+template <typename T>
+void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  bounds_bench_data<T> s(state);
+
+  state.add_element_count(s.needles);
+  state.add_global_memory_reads<T>(s.elements + s.needles);
+  state.add_global_memory_writes<std::ptrdiff_t>(s.needles);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    const auto env = cub_bench_env(alloc, launch);
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceFind::UpperBound,
+      "UpperBound failed",
+      s.range_ptr(),
+      static_cast<std::int64_t>(s.elements),
+      s.values_ptr(),
+      static_cast<std::int64_t>(s.needles),
+      s.output_ptr(),
+      less_t{},
+      env);
+  });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(integral_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("NeedlesRatio", {1, 25, 50});

cccl_upstream/cub/benchmarks/bench/find_bound/upper_bound_sorted_values.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+//! `cub::DeviceFind::UpperBoundSortedValues`: haystack and values (needles) sorted.
+
+#include <cub/device/device_find.cuh>
+
+#include <cstdint>
+
+#include <nvbench_helper.cuh>
+
+#include "find_bound_common.cuh"
+
+template <typename T>
+void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  bounds_bench_data<T> s(state);
+  s.sort_needles();
+
+  state.add_element_count(s.needles);
+  state.add_global_memory_reads<T>(s.elements + s.needles);
+  state.add_global_memory_writes<std::ptrdiff_t>(s.needles);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    const auto env = cub_bench_env(alloc, launch);
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceFind::UpperBoundSortedValues,
+      "UpperBoundSortedValues failed",
+      s.range_ptr(),
+      static_cast<std::int64_t>(s.elements),
+      s.values_ptr(),
+      static_cast<std::int64_t>(s.needles),
+      s.output_ptr(),
+      less_t{},
+      env);
+  });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(integral_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("NeedlesRatio", {1, 25, 50});

cccl_upstream/cub/benchmarks/bench/find_if/base.cu

@@ -0,0 +1,77 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/device/device_find.cuh>
+
+#include <thrust/count.h>
+#include <thrust/detail/internal_functional.h>
+#include <thrust/find.h>
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_LOAD ld 0:2:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK_POW2 tpb 6:10:1
+
+#if !TUNE_BASE
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_LDG
+#  else // TUNE_LOAD == 2
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+
+template <typename T>
+struct bench_policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(::cuda::compute_capability) const
+    -> cub::detail::find::find_policy
+  {
+    return cub::detail::find::find_policy{
+      (1 << TUNE_THREADS_PER_BLOCK_POW2), cub::Nominal4BItemsToItems<T>(TUNE_ITEMS_PER_THREAD), 4, TUNE_LOAD_MODIFIER};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T, typename OffsetT>
+void find_if(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  T val = 1;
+  // set up input
+  const auto elements       = static_cast<OffsetT>(state.get_int64("Elements"));
+  const auto common_prefix  = state.get_float64("MismatchAt");
+  const auto mismatch_point = static_cast<OffsetT>(elements * common_prefix);
+
+  thrust::device_vector<T> dinput(elements, thrust::no_init);
+  thrust::fill(dinput.begin(), dinput.begin() + mismatch_point, 0);
+  thrust::fill(dinput.begin() + mismatch_point, dinput.end(), val);
+  thrust::device_vector<OffsetT> d_result(1, thrust::no_init);
+
+  state.add_global_memory_reads<T>(mismatch_point);
+  state.add_global_memory_writes<OffsetT>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<T>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceFind::FindIf,
+      "FindIf failed",
+      thrust::raw_pointer_cast(dinput.data()),
+      thrust::raw_pointer_cast(d_result.data()),
+      cuda::equal_to_value<T>(val),
+      static_cast<OffsetT>(dinput.size()),
+      env);
+  });
+}
+
+NVBENCH_BENCH_TYPES(find_if, NVBENCH_TYPE_AXES(fundamental_types, offset_types))
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_float64_axis("MismatchAt", std::vector{1.0, 0.5, 0.0});

cccl_upstream/cub/benchmarks/bench/for_each/base.cu

@@ -0,0 +1,54 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_for.cuh>
+
+#include <nvbench_helper.cuh>
+
+template <class T>
+struct op_t
+{
+  int* d_count{};
+
+  __device__ void operator()(T val) const
+  {
+    if (val == T{})
+    {
+      atomicAdd(d_count, 1);
+    }
+  }
+};
+
+template <class T, class OffsetT>
+void for_each(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using input_it_t  = const T*;
+  using output_it_t = int*;
+  using offset_t    = OffsetT;
+
+  const auto elements = static_cast<offset_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<T> in(elements, T{42});
+
+  input_it_t d_in = thrust::raw_pointer_cast(in.data());
+  // `d_out` exists for visibility
+  // All inputs are equal to `42`, while the operator is searching for `0`.
+  // If the operator finds `0` in the input sequence, it's an issue leading to a segfault.
+  output_it_t d_out = nullptr;
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+
+  op_t<T> op{d_out};
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(alloc, launch);
+    _CCCL_TRY_CUDA_API(cub::DeviceFor::ForEachN, "ForEachN failed", d_in, elements, op, env);
+  });
+}
+
+NVBENCH_BENCH_TYPES(for_each, NVBENCH_TYPE_AXES(fundamental_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/for_each/copy.cu

@@ -0,0 +1,51 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_for.cuh>
+
+#include <nvbench_helper.cuh>
+
+template <class T>
+struct op_t
+{
+  int* d_count{};
+
+  __device__ void operator()(T val) const
+  {
+    if (val == T{})
+    {
+      atomicAdd(d_count, 1);
+    }
+  }
+};
+
+template <class T, class OffsetT>
+void for_each(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using input_it_t  = const T*;
+  using output_it_t = int*;
+  using offset_t    = OffsetT;
+
+  const auto elements = static_cast<offset_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<T> in(elements, T{42});
+
+  input_it_t d_in   = thrust::raw_pointer_cast(in.data());
+  output_it_t d_out = nullptr;
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+
+  op_t<T> op{d_out};
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(alloc, launch);
+    _CCCL_TRY_CUDA_API(cub::DeviceFor::ForEachCopyN, "ForEachCopyN failed", d_in, elements, op, env);
+  });
+}
+
+NVBENCH_BENCH_TYPES(for_each, NVBENCH_TYPE_AXES(fundamental_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/for_each/extents.cu

@@ -0,0 +1,62 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include <cub/device/device_for.cuh>
+
+#include <cuda/cmath>
+#include <cuda/std/mdspan>
+
+#include <nvbench_helper.cuh>
+
+template <typename T, typename OffsetT>
+struct op_t
+{
+  using ext_t = cuda::std::dextents<OffsetT, 2>;
+  cuda::std::mdspan<T, ext_t> temp_in;
+  cuda::std::mdspan<T, ext_t> temp_out;
+
+  __device__ void operator()(OffsetT, OffsetT row, OffsetT column) const
+  {
+    if (row > 0 && column > 0 && row < temp_in.extent(0) - 1 && column < temp_in.extent(1) - 1)
+    {
+      T d2tdx2              = temp_in(row, column - 1) - 2 * temp_in(row, column) + temp_in(row, column + 1);
+      T d2tdy2              = temp_in(row - 1, column) - 2 * temp_in(row, column) + temp_in(row + 1, column);
+      temp_out(row, column) = temp_in(row, column) + 0.2f * (d2tdx2 + d2tdy2);
+    }
+    else
+    {
+      temp_out(row, column) = temp_in(row, column);
+    }
+  }
+};
+
+template <class T, class OffsetT>
+void for_each_in_extents(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using it_t          = T*;
+  using ext_t         = cuda::std::dextents<OffsetT, 2>;
+  const auto elements = static_cast<OffsetT>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<T> in(elements, T{42});
+  thrust::device_vector<T> out(elements);
+  it_t d_in  = thrust::raw_pointer_cast(in.data());
+  it_t d_out = thrust::raw_pointer_cast(out.data());
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  auto elements_1D = ::cuda::isqrt(elements);
+  ext_t ext{elements_1D, elements_1D};
+  cuda::std::mdspan<T, ext_t> temp_in{d_in, ext};
+  cuda::std::mdspan<T, ext_t> temp_out{d_out, ext};
+  op_t<T, OffsetT> op{temp_in, temp_out};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    _CCCL_TRY_CUDA_API(cub::DeviceFor::ForEachInExtents, "ForEachInExtents failed", ext, op, launch.get_stream());
+  });
+}
+
+NVBENCH_BENCH_TYPES(for_each_in_extents, NVBENCH_TYPE_AXES(fundamental_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/histogram/even.cu

@@ -0,0 +1,85 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <nvbench_helper.cuh>
+
+#include "histogram_common.cuh"
+
+// %RANGE% TUNE_ITEMS ipt 4:28:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_RLE_COMPRESS rle 0:1:1
+// %RANGE% TUNE_WORK_STEALING ws 0:1:1
+// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1
+// %RANGE% TUNE_LOAD ld 0:2:1
+// %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1
+// %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1
+
+template <typename SampleT, typename CounterT, typename OffsetT>
+static void even(nvbench::state& state, nvbench::type_list<SampleT, CounterT, OffsetT>)
+{
+  const auto entropy   = str_to_entropy(state.get_string("Entropy"));
+  const auto elements  = state.get_int64("Elements{io}");
+  const auto num_bins  = state.get_int64("Bins");
+  const int num_levels = static_cast<int>(num_bins) + 1;
+
+  // Skip invalid configurations where LevelT (= SampleT) cannot represent the number of bins
+  if constexpr (cuda::std::is_integral_v<SampleT>)
+  {
+    if (num_bins > static_cast<int64_t>(cuda::std::numeric_limits<SampleT>::max()))
+    {
+      state.skip("Number of bins exceeds what LevelT (= SampleT) can represent");
+      return;
+    }
+  }
+
+  const SampleT lower_level = 0;
+  const SampleT upper_level = get_upper_level<SampleT>(num_bins, elements);
+
+  thrust::device_vector<SampleT> input = generate(elements, entropy, lower_level, upper_level);
+  thrust::device_vector<CounterT> hist(num_bins);
+
+  SampleT* d_input      = thrust::raw_pointer_cast(input.data());
+  CounterT* d_histogram = thrust::raw_pointer_cast(hist.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<SampleT>(elements);
+  state.add_global_memory_writes<CounterT>(num_bins);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<key_t, 1, 1>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceHistogram::HistogramEven,
+      "HistogramEven failed",
+      d_input,
+      d_histogram,
+      num_levels,
+      lower_level,
+      upper_level,
+      static_cast<OffsetT>(elements),
+      env);
+  });
+}
+
+using counter_types     = nvbench::type_list<int32_t>;
+using some_offset_types = nvbench::type_list<int32_t>;
+
+#ifdef TUNE_SampleT
+using sample_types = nvbench::type_list<TUNE_SampleT>;
+#else // !defined(TUNE_SampleT)
+using sample_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double>;
+#endif // TUNE_SampleT
+
+NVBENCH_BENCH_TYPES(even, NVBENCH_TYPE_AXES(sample_types, counter_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"SampleT{ct}", "CounterT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("Bins", {32, 128, 2048, 2097152})
+  .add_string_axis("Entropy", {"0.201", "1.000"});

cccl_upstream/cub/benchmarks/bench/histogram/histogram_common.cuh

@@ -0,0 +1,78 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#pragma once
+
+#include <cub/device/device_histogram.cuh>
+
+#include <cuda/std/type_traits>
+
+#if !TUNE_BASE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_LDG
+#  else // TUNE_LOAD == 2
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+
+#  define TUNE_VEC_SIZE (1 << TUNE_VEC_SIZE_POW)
+
+#  if TUNE_MEM_PREFERENCE == 0
+constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::GMEM;
+#  elif TUNE_MEM_PREFERENCE == 1
+constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::SMEM;
+#  else // TUNE_MEM_PREFERENCE == 2
+constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::BLEND;
+#  endif // TUNE_MEM_PREFERENCE
+
+#  if TUNE_LOAD_ALGORITHM_ID == 0
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_DIRECT
+#  elif TUNE_LOAD_ALGORITHM_ID == 1
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_WARP_TRANSPOSE
+#  else
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_STRIPED
+#  endif // TUNE_LOAD_ALGORITHM_ID
+
+template <typename SampleT, int NUM_CHANNELS, int NUM_ACTIVE_CHANNELS>
+struct bench_policy_selector
+{
+  _CCCL_API constexpr auto operator()(::cuda::compute_capability) const -> cub::HistogramPolicy
+  {
+    constexpr cub::BlockLoadAlgorithm load_algorithm =
+      (TUNE_LOAD_ALGORITHM == cub::BLOCK_LOAD_STRIPED)
+        ? (NUM_CHANNELS == 1 ? cub::BLOCK_LOAD_STRIPED : cub::BLOCK_LOAD_DIRECT)
+        : TUNE_LOAD_ALGORITHM;
+
+    return {TUNE_THREADS,
+            TUNE_ITEMS,
+            TUNE_VEC_SIZE,
+            load_algorithm,
+            TUNE_LOAD_MODIFIER,
+            TUNE_RLE_COMPRESS,
+            MEM_PREFERENCE,
+            TUNE_WORK_STEALING,
+            2048}; // TODO(bgruber): make tunable
+  }
+};
+#endif // !TUNE_BASE
+
+template <class SampleT, class OffsetT>
+SampleT get_upper_level(OffsetT bins, OffsetT elements)
+{
+  if constexpr (cuda::std::is_integral_v<SampleT>)
+  {
+    if constexpr (sizeof(SampleT) < sizeof(OffsetT))
+    {
+      const SampleT max_key = ::cuda::std::numeric_limits<SampleT>::max();
+      return static_cast<SampleT>(std::min(bins, static_cast<OffsetT>(max_key)));
+    }
+    else
+    {
+      return static_cast<SampleT>(bins);
+    }
+  }
+
+  return static_cast<SampleT>(elements);
+}

cccl_upstream/cub/benchmarks/bench/histogram/multi/even.cu

@@ -0,0 +1,98 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <nvbench_helper.cuh>
+
+#include "../histogram_common.cuh"
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_RLE_COMPRESS rle 0:1:1
+// %RANGE% TUNE_WORK_STEALING ws 0:1:1
+// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1
+// %RANGE% TUNE_LOAD ld 0:2:1
+// %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1
+// %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1
+
+template <typename SampleT, typename CounterT, typename OffsetT>
+static void even(nvbench::state& state, nvbench::type_list<SampleT, CounterT, OffsetT>)
+{
+  constexpr int num_channels        = 4;
+  constexpr int num_active_channels = 3;
+
+  const auto entropy     = str_to_entropy(state.get_string("Entropy"));
+  const auto elements    = state.get_int64("Elements{io}");
+  const auto num_bins    = state.get_int64("Bins");
+  const int num_levels_r = static_cast<int>(num_bins) + 1;
+  const int num_levels_g = num_levels_r;
+  const int num_levels_b = num_levels_g;
+
+  // Skip invalid configurations where LevelT (= SampleT) cannot represent the number of bins
+  if constexpr (cuda::std::is_integral_v<SampleT>)
+  {
+    if (num_bins > static_cast<int64_t>(cuda::std::numeric_limits<SampleT>::max()))
+    {
+      state.skip("Number of bins exceeds what LevelT (= SampleT) can represent");
+      return;
+    }
+  }
+
+  const SampleT lower_level_r = 0;
+  const SampleT upper_level_r = get_upper_level<SampleT>(num_bins, elements);
+  const SampleT lower_level_g = lower_level_r;
+  const SampleT upper_level_g = upper_level_r;
+  const SampleT lower_level_b = lower_level_g;
+  const SampleT upper_level_b = upper_level_g;
+
+  thrust::device_vector<CounterT> hist_r(num_bins);
+  thrust::device_vector<CounterT> hist_g(num_bins);
+  thrust::device_vector<CounterT> hist_b(num_bins);
+  thrust::device_vector<SampleT> input = generate(elements * num_channels, entropy, lower_level_r, upper_level_r);
+
+  SampleT* d_input        = thrust::raw_pointer_cast(input.data());
+  CounterT* d_histogram_r = thrust::raw_pointer_cast(hist_r.data());
+  CounterT* d_histogram_g = thrust::raw_pointer_cast(hist_g.data());
+  CounterT* d_histogram_b = thrust::raw_pointer_cast(hist_b.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<SampleT>(elements * num_active_channels);
+  state.add_global_memory_writes<CounterT>(num_bins * num_active_channels);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<key_t, num_channels, num_active_channels>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      (cub::DeviceHistogram::MultiHistogramEven<num_channels, num_active_channels>),
+      "MultiHistogramEven failed",
+      d_input,
+      cuda::std::array<CounterT*, num_active_channels>{d_histogram_r, d_histogram_g, d_histogram_b},
+      cuda::std::array<int, num_active_channels>{num_levels_r, num_levels_g, num_levels_b},
+      cuda::std::array<SampleT, num_active_channels>{lower_level_r, lower_level_g, lower_level_b},
+      cuda::std::array<SampleT, num_active_channels>{upper_level_r, upper_level_g, upper_level_b},
+      static_cast<OffsetT>(elements),
+      env);
+  });
+}
+
+using counter_types     = nvbench::type_list<int32_t>;
+using some_offset_types = nvbench::type_list<int32_t>;
+
+#ifdef TUNE_SampleT
+using sample_types = nvbench::type_list<TUNE_SampleT>;
+#else // !defined(TUNE_SampleT)
+using sample_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double>;
+#endif // TUNE_SampleT
+
+NVBENCH_BENCH_TYPES(even, NVBENCH_TYPE_AXES(sample_types, counter_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"SampleT{ct}", "CounterT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("Bins", {32, 128, 2048, 2097152})
+  .add_string_axis("Entropy", {"0.201", "1.000"});

cccl_upstream/cub/benchmarks/bench/histogram/multi/range.cu

@@ -0,0 +1,97 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/sequence.h>
+
+#include <nvbench_helper.cuh>
+
+#include "../histogram_common.cuh"
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_RLE_COMPRESS rle 0:1:1
+// %RANGE% TUNE_WORK_STEALING ws 0:1:1
+// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1
+// %RANGE% TUNE_LOAD ld 0:2:1
+// %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1
+// %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1
+
+template <typename SampleT, typename CounterT, typename OffsetT>
+static void range(nvbench::state& state, nvbench::type_list<SampleT, CounterT, OffsetT>)
+{
+  constexpr int num_channels        = 4;
+  constexpr int num_active_channels = 3;
+
+  const auto entropy     = str_to_entropy(state.get_string("Entropy"));
+  const auto elements    = state.get_int64("Elements{io}");
+  const auto num_bins    = state.get_int64("Bins");
+  const int num_levels_r = static_cast<int>(num_bins) + 1;
+  const int num_levels_g = num_levels_r;
+  const int num_levels_b = num_levels_g;
+
+  const SampleT lower_level = 0;
+  const SampleT upper_level = get_upper_level<SampleT>(num_bins, elements);
+
+  SampleT step = (upper_level - lower_level) / num_bins;
+  thrust::device_vector<SampleT> levels_r(num_bins + 1);
+
+  // TODO Extract sequence to the helper TU
+  thrust::sequence(levels_r.begin(), levels_r.end(), lower_level, step);
+  thrust::device_vector<SampleT> levels_g = levels_r;
+  thrust::device_vector<SampleT> levels_b = levels_g;
+
+  SampleT* d_levels_r = thrust::raw_pointer_cast(levels_r.data());
+  SampleT* d_levels_g = thrust::raw_pointer_cast(levels_g.data());
+  SampleT* d_levels_b = thrust::raw_pointer_cast(levels_b.data());
+
+  thrust::device_vector<CounterT> hist_r(num_bins);
+  thrust::device_vector<CounterT> hist_g(num_bins);
+  thrust::device_vector<CounterT> hist_b(num_bins);
+  thrust::device_vector<SampleT> input = generate(elements * num_channels, entropy, lower_level, upper_level);
+
+  SampleT* d_input        = thrust::raw_pointer_cast(input.data());
+  CounterT* d_histogram_r = thrust::raw_pointer_cast(hist_r.data());
+  CounterT* d_histogram_g = thrust::raw_pointer_cast(hist_g.data());
+  CounterT* d_histogram_b = thrust::raw_pointer_cast(hist_b.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<SampleT>(elements * num_active_channels);
+  state.add_global_memory_writes<CounterT>(num_bins * num_active_channels);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<key_t, num_channels, num_active_channels>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      (cub::DeviceHistogram::MultiHistogramRange<num_channels, num_active_channels>),
+      "MultiHistogramRange failed",
+      d_input,
+      cuda::std::array<CounterT*, num_active_channels>{d_histogram_r, d_histogram_g, d_histogram_b},
+      cuda::std::array<int, num_active_channels>{num_levels_r, num_levels_g, num_levels_b},
+      cuda::std::array<const SampleT*, num_active_channels>{d_levels_r, d_levels_g, d_levels_b},
+      static_cast<OffsetT>(elements),
+      env);
+  });
+}
+
+using counter_types     = nvbench::type_list<int32_t>;
+using some_offset_types = nvbench::type_list<int32_t>;
+
+#ifdef TUNE_SampleT
+using sample_types = nvbench::type_list<TUNE_SampleT>;
+#else // !defined(TUNE_SampleT)
+using sample_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double>;
+#endif // TUNE_SampleT
+
+NVBENCH_BENCH_TYPES(range, NVBENCH_TYPE_AXES(sample_types, counter_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"SampleT{ct}", "CounterT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("Bins", {32, 128, 2048, 2097152})
+  .add_string_axis("Entropy", {"0.201", "1.000"});

cccl_upstream/cub/benchmarks/bench/histogram/range.cu

@@ -0,0 +1,83 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/sequence.h>
+
+#include <nvbench_helper.cuh>
+
+#include "histogram_common.cuh"
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_RLE_COMPRESS rle 0:1:1
+// %RANGE% TUNE_WORK_STEALING ws 0:1:1
+// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1
+// %RANGE% TUNE_LOAD ld 0:2:1
+// %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1
+// %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1
+
+template <typename SampleT, typename CounterT, typename OffsetT>
+static void range(nvbench::state& state, nvbench::type_list<SampleT, CounterT, OffsetT>)
+{
+  const auto entropy   = str_to_entropy(state.get_string("Entropy"));
+  const auto elements  = state.get_int64("Elements{io}");
+  const auto num_bins  = state.get_int64("Bins");
+  const int num_levels = static_cast<int>(num_bins) + 1;
+
+  const SampleT lower_level = 0;
+  const SampleT upper_level = get_upper_level<SampleT>(num_bins, elements);
+
+  SampleT step = (upper_level - lower_level) / num_bins;
+  thrust::device_vector<SampleT> levels(num_bins + 1);
+
+  // TODO Extract sequence to the helper TU
+  thrust::sequence(levels.begin(), levels.end(), lower_level, step);
+  SampleT* d_levels = thrust::raw_pointer_cast(levels.data());
+
+  thrust::device_vector<SampleT> input = generate(elements, entropy, lower_level, upper_level);
+  thrust::device_vector<CounterT> hist(num_bins);
+
+  SampleT* d_input      = thrust::raw_pointer_cast(input.data());
+  CounterT* d_histogram = thrust::raw_pointer_cast(hist.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<SampleT>(elements);
+  state.add_global_memory_writes<CounterT>(num_bins);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<key_t, 1, 1>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceHistogram::HistogramRange,
+      "HistogramRange failed",
+      d_input,
+      d_histogram,
+      num_levels,
+      d_levels,
+      static_cast<OffsetT>(elements),
+      env);
+  });
+}
+
+using counter_types     = nvbench::type_list<int32_t>;
+using some_offset_types = nvbench::type_list<int32_t>;
+
+#ifdef TUNE_SampleT
+using sample_types = nvbench::type_list<TUNE_SampleT>;
+#else // !defined(TUNE_SampleT)
+using sample_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double>;
+#endif // TUNE_SampleT
+
+NVBENCH_BENCH_TYPES(range, NVBENCH_TYPE_AXES(sample_types, counter_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"SampleT{ct}", "CounterT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("Bins", {32, 128, 2048, 2097152})
+  .add_string_axis("Entropy", {"0.201", "1.000"});

cccl_upstream/cub/benchmarks/bench/merge/keys.cu

@@ -0,0 +1,78 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include <cub/device/device_merge.cuh>
+
+#include <thrust/detail/raw_pointer_cast.h>
+
+#include <cuda/std/utility>
+
+#include <cstdint>
+
+#include <nvbench_helper.cuh>
+
+#include "merge_common.cuh"
+
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:3:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK_POW2 tpb 6:10:1
+
+template <typename KeyT>
+void keys(nvbench::state& state, nvbench::type_list<KeyT>)
+{
+  using offset_t     = int64_t;
+  using compare_op_t = less_t;
+
+  // Retrieve axis parameters
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  const auto num_items_lhs  = elements / 2;
+  const auto num_items_rhs  = elements - num_items_lhs;
+  auto [keys_lhs, keys_rhs] = generate_lhs_rhs<KeyT>(num_items_lhs, num_items_rhs, entropy);
+
+  thrust::device_vector<KeyT> keys_out(elements, thrust::no_init);
+  KeyT* d_keys_lhs = thrust::raw_pointer_cast(keys_lhs.data());
+  KeyT* d_keys_rhs = thrust::raw_pointer_cast(keys_rhs.data());
+  KeyT* d_keys_out = thrust::raw_pointer_cast(keys_out.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_writes<KeyT>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<key_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceMerge::MergeKeys,
+      "MergePairs failed",
+      d_keys_lhs,
+      static_cast<offset_t>(num_items_lhs),
+      d_keys_rhs,
+      static_cast<offset_t>(num_items_rhs),
+      d_keys_out,
+      compare_op_t{},
+      env);
+  });
+}
+
+#ifdef TUNE_KeyT
+using key_types = nvbench::type_list<TUNE_KeyT>;
+#else // !defined(TUNE_KeyT)
+using key_types = fundamental_types;
+#endif // TUNE_KeyT
+
+NVBENCH_BENCH_TYPES(keys, NVBENCH_TYPE_AXES(key_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"});

cccl_upstream/cub/benchmarks/bench/merge/merge_common.cuh

@@ -0,0 +1,135 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+#include <thrust/copy.h>
+#include <thrust/count.h>
+#include <thrust/device_vector.h>
+#include <thrust/sort.h>
+
+#include <cuda/iterator>
+
+#include <nvbench_helper.cuh>
+
+#if !TUNE_BASE
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#    define TUNE_USE_BL2SH     false
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_LDG
+#    define TUNE_USE_BL2SH     false
+#  elif TUNE_LOAD == 2
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#    define TUNE_USE_BL2SH     false
+#  else // TUNE_LOAD == 3
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#    define TUNE_USE_BL2SH     true
+#  endif // TUNE_LOAD
+
+template <typename KeyT>
+struct bench_policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::MergePolicy
+  {
+    return cub::MergePolicy{
+      (1 << TUNE_THREADS_PER_BLOCK_POW2),
+      cub::Nominal4BItemsToItems<KeyT>(TUNE_ITEMS_PER_THREAD),
+      TUNE_LOAD_MODIFIER,
+      TUNE_TRANSPOSE == 0 ? cub::BLOCK_STORE_DIRECT : cub::BLOCK_STORE_WARP_TRANSPOSE,
+      TUNE_USE_BL2SH};
+  }
+};
+#endif // TUNE_BASE
+
+struct select_if_less_than_t
+{
+  bool negate;
+  uint8_t threshold;
+
+  __device__ __forceinline__ bool operator()(uint8_t val) const
+  {
+    return negate ? !(val < threshold) : val < threshold;
+  }
+};
+
+template <typename OffsetT>
+struct write_pivot_point_t
+{
+  OffsetT threshold;
+  OffsetT* pivot_point;
+
+  __device__ void operator()(OffsetT output_index, OffsetT input_index) const
+  {
+    if (output_index == threshold)
+    {
+      *pivot_point = input_index;
+    }
+  }
+};
+
+template <typename KeyT>
+std::pair<thrust::device_vector<KeyT>, thrust::device_vector<KeyT>>
+generate_lhs_rhs(std::size_t num_items_lhs, std::size_t num_items_rhs, bit_entropy entropy)
+{
+  using offset_t = std::size_t;
+
+  const auto elements = num_items_lhs + num_items_rhs;
+
+  // We generate data distributions in the range [0, 255], which, with lower entropy, get skewed towards 0.
+  // We use this to generate increasingly large *consecutive* segments of data that are getting selected from the lhs
+  thrust::device_vector<uint8_t> rnd_selector_val = generate(elements, entropy);
+  uint8_t threshold                               = 128;
+  select_if_less_than_t select_lhs_op{false, threshold};
+  select_if_less_than_t select_rhs_op{true, threshold};
+
+  // The following algorithm only works under the precondition that there's at least 50% of the data in the lhs
+  // If that's not the case, we simply swap the logic for selecting into lhs and rhs
+  const auto num_items_selected_into_lhs =
+    static_cast<offset_t>(thrust::count_if(rnd_selector_val.begin(), rnd_selector_val.end(), select_lhs_op));
+  if (num_items_selected_into_lhs < num_items_lhs)
+  {
+    using ::cuda::std::swap;
+    swap(select_lhs_op, select_rhs_op);
+  }
+
+  // We want lhs and rhs to be of equal size. We also want to have skewed distributions, such that we put different
+  // workloads on the binary search part. For this reason, we identify the index from the input, referred to as pivot
+  // point, after which the lhs is "full". We compose the rhs by selecting all items up to the pivot point that were not
+  // selected for lhs and *all* items after the pivot point.
+  constexpr std::size_t num_pivot_points = 1;
+  thrust::device_vector<offset_t> pivot_point(num_pivot_points);
+  auto counting_it            = thrust::make_counting_iterator(offset_t{0});
+  using counting_difference_t = typename decltype(counting_it)::difference_type;
+  thrust::copy_if(
+    counting_it,
+    counting_it + static_cast<counting_difference_t>(elements),
+    rnd_selector_val.begin(),
+    cuda::make_tabulate_output_iterator(write_pivot_point_t<offset_t>{
+      static_cast<offset_t>(num_items_lhs), thrust::raw_pointer_cast(pivot_point.data())}),
+    select_lhs_op);
+
+  thrust::device_vector<KeyT> keys_lhs(num_items_lhs);
+  thrust::device_vector<KeyT> keys_rhs(num_items_rhs);
+
+  thrust::device_vector<KeyT> increasing_input = generate(elements);
+  thrust::sort(increasing_input.begin(), increasing_input.end());
+
+  offset_t pivot_point_val = pivot_point[0];
+  auto const end_lhs       = thrust::copy_if(
+    increasing_input.cbegin(),
+    increasing_input.cbegin() + pivot_point_val,
+    rnd_selector_val.cbegin(),
+    keys_lhs.begin(),
+    select_lhs_op);
+
+  auto const end_rhs = thrust::copy_if(
+    increasing_input.cbegin(),
+    increasing_input.cbegin() + pivot_point_val,
+    rnd_selector_val.cbegin(),
+    keys_rhs.begin(),
+    select_rhs_op);
+  thrust::copy(increasing_input.cbegin() + pivot_point_val, increasing_input.cbegin() + elements, end_rhs);
+
+  return {keys_lhs, keys_rhs};
+}

cccl_upstream/cub/benchmarks/bench/merge/pairs.cu

@@ -0,0 +1,103 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include <cub/device/device_merge.cuh>
+
+#include <thrust/detail/raw_pointer_cast.h>
+
+#include <cuda/std/utility>
+
+#include <cstdint>
+
+#include <nvbench_helper.cuh>
+
+#include "merge_common.cuh"
+
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:3:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK_POW2 tpb 6:10:1
+
+template <typename KeyT, typename ValueT>
+void pairs(nvbench::state& state, nvbench::type_list<KeyT, ValueT>)
+{
+  using offset_t     = int64_t;
+  using compare_op_t = less_t;
+
+  // Retrieve axis parameters
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  const auto num_items_lhs = elements / 2;
+  const auto num_items_rhs = elements - num_items_lhs;
+
+  thrust::device_vector<KeyT> keys_out(elements, thrust::no_init);
+  thrust::device_vector<ValueT> values_lhs(num_items_lhs, thrust::no_init);
+  thrust::device_vector<ValueT> values_rhs(num_items_rhs, thrust::no_init);
+  thrust::device_vector<ValueT> values_out(elements, thrust::no_init);
+
+  auto [keys_lhs, keys_rhs] = generate_lhs_rhs<KeyT>(num_items_lhs, num_items_rhs, entropy);
+
+  KeyT* d_keys_lhs     = thrust::raw_pointer_cast(keys_lhs.data());
+  KeyT* d_keys_rhs     = thrust::raw_pointer_cast(keys_rhs.data());
+  KeyT* d_keys_out     = thrust::raw_pointer_cast(keys_out.data());
+  ValueT* d_values_lhs = thrust::raw_pointer_cast(values_lhs.data());
+  ValueT* d_values_rhs = thrust::raw_pointer_cast(values_rhs.data());
+  ValueT* d_values_out = thrust::raw_pointer_cast(values_out.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<KeyT>(elements);
+  state.add_global_memory_writes<ValueT>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<key_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceMerge::MergePairs,
+      "MergePairs failed",
+      d_keys_lhs,
+      d_values_lhs,
+      static_cast<offset_t>(num_items_lhs),
+      d_keys_rhs,
+      d_values_rhs,
+      static_cast<offset_t>(num_items_rhs),
+      d_keys_out,
+      d_values_out,
+      compare_op_t{},
+      env);
+  });
+}
+
+#ifdef TUNE_KeyT
+using key_types = nvbench::type_list<TUNE_KeyT>;
+#else // !defined(TUNE_KeyT)
+using key_types = fundamental_types;
+#endif // TUNE_KeyT
+
+#ifdef TUNE_ValueT
+using value_types = nvbench::type_list<TUNE_ValueT>;
+#else // !defined(TUNE_ValueT)
+using value_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t
+#  if _CCCL_HAS_INT128()
+// nvcc currently hangs for __int128 value type with the fallback policy of {CTA: 64, IPT: 1}. NVBug 4384075
+//  ,
+//  int128_t
+#  endif
+                                       >;
+#endif // TUNE_ValueT
+
+NVBENCH_BENCH_TYPES(pairs, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"});

cccl_upstream/cub/benchmarks/bench/merge_sort/keys.cu

@@ -0,0 +1,80 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/device_merge_sort.cuh>
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:2:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK_POW2 tpb 6:10:1
+
+#ifndef TUNE_BASE
+#  define TUNE_THREADS_PER_BLOCK (1 << TUNE_THREADS_PER_BLOCK_POW2)
+#endif // TUNE_BASE
+
+#if !TUNE_BASE
+template <typename KeyT>
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::MergeSortPolicy
+  {
+    return cub::MergeSortPolicy{
+      TUNE_THREADS_PER_BLOCK,
+      cub::Nominal4BItemsToItems<KeyT>(TUNE_ITEMS_PER_THREAD),
+      (TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE),
+      (TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : (TUNE_LOAD == 1 ? cub::LOAD_LDG : cub::LOAD_CA)),
+      (TUNE_TRANSPOSE == 0 ? cub::BLOCK_STORE_DIRECT : cub::BLOCK_STORE_WARP_TRANSPOSE)};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T, typename OffsetT>
+void keys(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using key_t        = T;
+  using compare_op_t = less_t;
+
+  // Retrieve axis parameters
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  thrust::device_vector<T> buffer_1 = generate(elements, entropy);
+  thrust::device_vector<T> buffer_2(elements);
+
+  key_t* d_buffer_1 = thrust::raw_pointer_cast(buffer_1.data());
+  key_t* d_buffer_2 = thrust::raw_pointer_cast(buffer_2.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<key_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceMergeSort::SortKeysCopy,
+      "SortKeysCopy failed",
+      d_buffer_1,
+      d_buffer_2,
+      static_cast<OffsetT>(elements),
+      compare_op_t{},
+      env);
+  });
+}
+
+NVBENCH_BENCH_TYPES(keys, NVBENCH_TYPE_AXES(all_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"});

cccl_upstream/cub/benchmarks/bench/merge_sort/pairs.cu

@@ -0,0 +1,107 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/device_merge_sort.cuh>
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:2:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK_POW2 tpb 6:10:1
+
+#ifndef TUNE_BASE
+#  define TUNE_THREADS_PER_BLOCK (1 << TUNE_THREADS_PER_BLOCK_POW2)
+#endif
+
+#if !TUNE_BASE
+template <typename KeyT>
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::MergeSortPolicy
+  {
+    return cub::MergeSortPolicy{
+      TUNE_THREADS_PER_BLOCK,
+      cub::Nominal4BItemsToItems<KeyT>(TUNE_ITEMS_PER_THREAD),
+      (TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE),
+      (TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : (TUNE_LOAD == 1 ? cub::LOAD_LDG : cub::LOAD_CA)),
+      (TUNE_TRANSPOSE == 0 ? cub::BLOCK_STORE_DIRECT : cub::BLOCK_STORE_WARP_TRANSPOSE)};
+  }
+};
+#endif // TUNE_BASE
+
+template <typename KeyT, typename ValueT, typename OffsetT>
+void pairs(nvbench::state& state, nvbench::type_list<KeyT, ValueT, OffsetT>)
+{
+  using key_t        = KeyT;
+  using value_t      = ValueT;
+  using compare_op_t = less_t;
+
+  // Retrieve axis parameters
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  thrust::device_vector<key_t> keys_buffer_1 = generate(elements, entropy);
+  thrust::device_vector<key_t> keys_buffer_2(elements);
+  thrust::device_vector<value_t> values_buffer_1(elements);
+  thrust::device_vector<value_t> values_buffer_2(elements);
+
+  key_t* d_keys_buffer_1     = thrust::raw_pointer_cast(keys_buffer_1.data());
+  key_t* d_keys_buffer_2     = thrust::raw_pointer_cast(keys_buffer_2.data());
+  value_t* d_values_buffer_1 = thrust::raw_pointer_cast(values_buffer_1.data());
+  value_t* d_values_buffer_2 = thrust::raw_pointer_cast(values_buffer_2.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<KeyT>(elements);
+  state.add_global_memory_writes<ValueT>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<key_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceMergeSort::SortPairsCopy,
+      "SortPairsCopy failed",
+      d_keys_buffer_1,
+      d_values_buffer_1,
+      d_keys_buffer_2,
+      d_values_buffer_2,
+      static_cast<OffsetT>(elements),
+      compare_op_t{},
+      env);
+  });
+}
+
+#ifdef TUNE_KeyT
+using key_types = nvbench::type_list<TUNE_KeyT>;
+#else // !defined(TUNE_KeyT)
+using key_types = all_types;
+#endif // TUNE_KeyT
+
+#ifdef TUNE_ValueT
+using value_types = nvbench::type_list<TUNE_ValueT>;
+#else // !defined(TUNE_ValueT)
+using value_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t
+#  if _CCCL_HAS_INT128()
+// nvcc currently hangs for __int128 value type with the fallback policy of {CTA: 64, IPT: 1}. NVBug 4384075
+//  ,
+//  int128_t
+#  endif
+                                       >;
+#endif // TUNE_ValueT
+
+NVBENCH_BENCH_TYPES(pairs, NVBENCH_TYPE_AXES(key_types, value_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"});

cccl_upstream/cub/benchmarks/bench/partition/flagged.cu

@@ -0,0 +1,133 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_partition.cuh>
+
+#include <thrust/count.h>
+
+#include <cuda/std/algorithm>
+#include <cuda/std/type_traits>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  else // TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+
+template <typename InputT>
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::PartitionPolicy
+  {
+    return {cub::PartitionAlgorithm::lookback,
+            {TUNE_THREADS_PER_BLOCK,
+             TUNE_ITEMS_PER_THREAD,
+             TUNE_LOAD_ALGORITHM,
+             TUNE_LOAD_MODIFIER,
+             cub::BLOCK_SCAN_WARP_SCANS,
+             lookback_delay_policy}};
+  }
+};
+#endif // TUNE_BASE
+
+template <typename FlagsItT, typename T, typename OffsetT>
+void init_output_partition_buffer(
+  FlagsItT d_flags,
+  OffsetT num_items,
+  T* d_out,
+  cub::detail::select::partition_distinct_output_t<T*, T*>& d_partition_out_buffer)
+{
+  const auto selected_elements = thrust::count(d_flags, d_flags + num_items, true);
+  d_partition_out_buffer = cub::detail::select::partition_distinct_output_t<T*, T*>{d_out, d_out + selected_elements};
+}
+
+template <typename FlagsItT, typename T, typename OffsetT>
+void init_output_partition_buffer(FlagsItT, OffsetT, T* d_out, T*& d_partition_out_buffer)
+{
+  d_partition_out_buffer = d_out;
+}
+
+template <typename T, typename OffsetT, typename UseDistinctPartitionT>
+void flagged(nvbench::state& state, nvbench::type_list<T, OffsetT, UseDistinctPartitionT>)
+{
+  using offset_t                             = OffsetT;
+  constexpr bool use_distinct_out_partitions = UseDistinctPartitionT::value;
+  using output_it_t                          = typename ::cuda::std::
+    conditional<use_distinct_out_partitions, cub::detail::select::partition_distinct_output_t<T*, T*>, T*>::type;
+
+  // Retrieve axis parameters
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  auto generator = generate(elements, entropy);
+
+  thrust::device_vector<T> in       = generator;
+  thrust::device_vector<bool> flags = generator;
+  thrust::device_vector<offset_t> num_selected(1);
+  thrust::device_vector<T> out(elements);
+
+  const T* d_in            = thrust::raw_pointer_cast(in.data());
+  const bool* d_flags      = thrust::raw_pointer_cast(flags.data());
+  offset_t* d_num_selected = thrust::raw_pointer_cast(num_selected.data());
+  output_it_t d_out{};
+  init_output_partition_buffer(flags.cbegin(), elements, thrust::raw_pointer_cast(out.data()), d_out);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_reads<bool>(elements);
+  state.add_global_memory_writes<T>(elements);
+  state.add_global_memory_writes<offset_t>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<T>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DevicePartition::Flagged,
+      "Flagged failed",
+      d_in,
+      d_flags,
+      d_out,
+      d_num_selected,
+      static_cast<offset_t>(elements),
+      env);
+  });
+}
+
+using ::cuda::std::false_type;
+using ::cuda::std::true_type;
+#ifdef TUNE_DistinctPartitions
+using distinct_partitions = nvbench::type_list<TUNE_DistinctPartitions>; // expands to "false_type" or "true_type"
+#else // !defined(TUNE_DistinctPartitions)
+using distinct_partitions = nvbench::type_list<false_type, true_type>;
+#endif // TUNE_DistinctPartitions
+
+NVBENCH_BENCH_TYPES(flagged, NVBENCH_TYPE_AXES(fundamental_types, offset_types, distinct_partitions))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}", "DistinctPartitions{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.000"});

cccl_upstream/cub/benchmarks/bench/partition/if.cu

@@ -0,0 +1,134 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_partition.cuh>
+
+#include <thrust/count.h>
+
+#include <cuda/std/algorithm>
+#include <cuda/std/type_traits>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  else // TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+
+template <typename InputT>
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::PartitionPolicy
+  {
+    return {cub::PartitionAlgorithm::lookback,
+            {TUNE_THREADS_PER_BLOCK,
+             TUNE_ITEMS_PER_THREAD,
+             TUNE_LOAD_ALGORITHM,
+             TUNE_LOAD_MODIFIER,
+             cub::BLOCK_SCAN_WARP_SCANS,
+             lookback_delay_policy}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename InItT, typename T, typename OffsetT, typename SelectOpT>
+void init_output_partition_buffer(
+  InItT d_in,
+  OffsetT num_items,
+  T* d_out,
+  SelectOpT select_op,
+  cub::detail::select::partition_distinct_output_t<T*, T*>& d_partition_out_buffer)
+{
+  const auto selected_elements = thrust::count_if(d_in, d_in + num_items, select_op);
+  d_partition_out_buffer = cub::detail::select::partition_distinct_output_t<T*, T*>{d_out, d_out + selected_elements};
+}
+
+template <typename InItT, typename T, typename OffsetT, typename SelectOpT>
+void init_output_partition_buffer(InItT, OffsetT, T* d_out, SelectOpT, T*& d_partition_out_buffer)
+{
+  d_partition_out_buffer = d_out;
+}
+
+template <typename T, typename OffsetT, typename UseDistinctPartitionT>
+void partition(nvbench::state& state, nvbench::type_list<T, OffsetT, UseDistinctPartitionT>)
+{
+  using select_op_t                          = less_then_t<T>;
+  using offset_t                             = OffsetT;
+  constexpr bool use_distinct_out_partitions = UseDistinctPartitionT::value;
+  using output_it_t                          = typename ::cuda::std::
+    conditional<use_distinct_out_partitions, cub::detail::select::partition_distinct_output_t<T*, T*>, T*>::type;
+
+  // Retrieve axis parameters
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  const T val = lerp_min_max<T>(entropy_to_probability(entropy));
+  select_op_t select_op{val};
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<offset_t> num_selected(1);
+
+  thrust::device_vector<T> out(elements);
+
+  const T* d_in            = thrust::raw_pointer_cast(in.data());
+  offset_t* d_num_selected = thrust::raw_pointer_cast(num_selected.data());
+  output_it_t d_out{};
+  init_output_partition_buffer(in.cbegin(), elements, thrust::raw_pointer_cast(out.data()), select_op, d_out);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+  state.add_global_memory_writes<offset_t>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<T>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DevicePartition::If,
+      "If failed",
+      d_in,
+      d_out,
+      d_num_selected,
+      static_cast<offset_t>(elements),
+      select_op,
+      env);
+  });
+}
+
+using ::cuda::std::false_type;
+using ::cuda::std::true_type;
+#ifdef TUNE_DistinctPartitions
+using distinct_partitions = nvbench::type_list<TUNE_DistinctPartitions>; // expands to "false_type" or "true_type"
+#else // !defined(TUNE_DistinctPartitions)
+using distinct_partitions = nvbench::type_list<false_type, true_type>;
+#endif // TUNE_DistinctPartitions
+
+NVBENCH_BENCH_TYPES(partition, NVBENCH_TYPE_AXES(fundamental_types, offset_types, distinct_partitions))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}", "DistinctPartitions{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.000"});

cccl_upstream/cub/benchmarks/bench/partition/three_way.cu

@@ -0,0 +1,99 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_partition.cuh>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+template <typename InputT>
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const
+    -> cub::ThreeWayPartitionPolicy
+  {
+    return {cub::ThreeWayPartitionAlgorithm::lookback,
+            {TUNE_THREADS_PER_BLOCK,
+             TUNE_ITEMS_PER_THREAD,
+             TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE,
+             cub::LOAD_DEFAULT,
+             cub::BLOCK_SCAN_WARP_SCANS,
+             lookback_delay_policy}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T, typename OffsetT>
+void partition(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using select_op_t = less_then_t<T>;
+  using offset_t    = OffsetT;
+
+  // Retrieve axis parameters
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  T min_val{};
+  T max_val = ::cuda::std::numeric_limits<T>::max();
+
+  T left_border  = max_val / 3;
+  T right_border = left_border * 2;
+
+  select_op_t select_op_1{left_border};
+  select_op_t select_op_2{right_border};
+
+  thrust::device_vector<T> in = generate(elements, entropy, min_val, max_val);
+  thrust::device_vector<offset_t> num_selected(2);
+  thrust::device_vector<T> out_1(elements);
+  thrust::device_vector<T> out_2(elements);
+  thrust::device_vector<T> out_3(elements);
+
+  const T* d_in            = thrust::raw_pointer_cast(in.data());
+  T* d_out_1               = thrust::raw_pointer_cast(out_1.data());
+  T* d_out_2               = thrust::raw_pointer_cast(out_2.data());
+  T* d_out_3               = thrust::raw_pointer_cast(out_3.data());
+  offset_t* d_num_selected = thrust::raw_pointer_cast(num_selected.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+  state.add_global_memory_writes<offset_t>(2);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<T>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DevicePartition::If,
+      "If three-way failed",
+      d_in,
+      d_out_1,
+      d_out_2,
+      d_out_3,
+      d_num_selected,
+      static_cast<offset_t>(elements),
+      select_op_1,
+      select_op_2,
+      env);
+  });
+}
+
+NVBENCH_BENCH_TYPES(partition, NVBENCH_TYPE_AXES(fundamental_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.000"});

cccl_upstream/cub/benchmarks/bench/radix_sort/keys.cu

@@ -0,0 +1,64 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <nvbench_helper.cuh>
+
+// %//RANGE//% TUNE_RADIX_BITS bits 8:9:1
+#define TUNE_RADIX_BITS 8
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+
+#include "policy_selector.h"
+
+template <typename T, typename OffsetT>
+void radix_sort_keys(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using value_t = cub::NullType;
+  if constexpr (!fits_in_default_shared_memory<T, value_t, OffsetT, cub::SortOrder::Ascending>())
+  {
+    return;
+  }
+
+  // Retrieve axis parameters
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  thrust::device_vector<T> buffer_1 = generate(elements, entropy);
+  thrust::device_vector<T> buffer_2(elements, thrust::no_init);
+
+  const T* d_buffer_1 = thrust::raw_pointer_cast(buffer_1.data());
+  T* d_buffer_2       = thrust::raw_pointer_cast(buffer_2.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<T, value_t, OffsetT>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceRadixSort::SortKeys,
+      "SortKeys failed",
+      d_buffer_1,
+      d_buffer_2,
+      static_cast<OffsetT>(elements),
+      0,
+      sizeof(T) * 8,
+      env);
+  });
+}
+
+NVBENCH_BENCH_TYPES(radix_sort_keys, NVBENCH_TYPE_AXES(fundamental_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.201"});

cccl_upstream/cub/benchmarks/bench/radix_sort/pairs.cu

@@ -0,0 +1,94 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_radix_sort.cuh>
+
+#include <nvbench_helper.cuh>
+
+// %//RANGE//% TUNE_RADIX_BITS bits 8:9:1
+#define TUNE_RADIX_BITS 8
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+
+#include "policy_selector.h"
+
+template <typename KeyT, typename ValueT, typename OffsetT>
+void radix_sort_values(nvbench::state& state, nvbench::type_list<KeyT, ValueT, OffsetT>)
+{
+  if constexpr (!fits_in_default_shared_memory<KeyT, ValueT, OffsetT, cub::SortOrder::Ascending>())
+  {
+    return;
+  }
+
+  // Retrieve axis parameters
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  thrust::device_vector<KeyT> keys_in = generate(elements, entropy);
+  thrust::device_vector<KeyT> keys_out(elements, thrust::no_init);
+  thrust::device_vector<ValueT> values_in = generate(elements);
+  thrust::device_vector<ValueT> values_out(elements, thrust::no_init);
+
+  const KeyT* d_keys_in     = thrust::raw_pointer_cast(keys_in.data());
+  KeyT* d_keys_out          = thrust::raw_pointer_cast(keys_out.data());
+  const ValueT* d_values_in = thrust::raw_pointer_cast(values_in.data());
+  ValueT* d_values_out      = thrust::raw_pointer_cast(values_out.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<KeyT>(elements);
+  state.add_global_memory_writes<ValueT>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<KeyT, ValueT, OffsetT>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceRadixSort::SortPairs,
+      "SortPairs failed",
+      d_keys_in,
+      d_keys_out,
+      d_values_in,
+      d_values_out,
+      static_cast<OffsetT>(elements),
+      0,
+      sizeof(KeyT) * 8,
+      env);
+  });
+}
+
+#ifdef TUNE_KeyT
+using key_types = nvbench::type_list<TUNE_KeyT>;
+#else // !defined(TUNE_KeyT)
+using key_types = integral_types;
+#endif // TUNE_KeyT
+
+#ifdef TUNE_ValueT
+using value_types = nvbench::type_list<TUNE_ValueT>;
+#else // !defined(Tune_ValueT)
+using value_types =
+  nvbench::type_list<int8_t,
+                     int16_t,
+                     int32_t,
+                     int64_t
+#  if _CCCL_HAS_INT128()
+                     ,
+                     int128_t
+#  endif
+                     >;
+#endif // TUNE_ValueT
+
+NVBENCH_BENCH_TYPES(radix_sort_values, NVBENCH_TYPE_AXES(key_types, value_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"});

cccl_upstream/cub/benchmarks/bench/radix_sort/policy_selector.h

@@ -0,0 +1,128 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_radix_sort.cuh>
+
+#if !TUNE_BASE
+template <typename KeyT, typename ValueT, typename OffsetT>
+struct policy_selector
+{
+  using DominantT = cuda::std::conditional_t<(sizeof(ValueT) > sizeof(KeyT)), ValueT, KeyT>;
+
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> ::cub::RadixSortPolicy
+  {
+    const auto onesweep = [] {
+      const auto scaled =
+        cub::detail::scale_reg_bound(TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, sizeof(DominantT));
+      return cub::RadixSortOnesweepPolicy{
+        scaled.threads_per_block,
+        scaled.items_per_thread,
+        cub::RADIX_SORT_STORE_DIRECT,
+        cub::RADIX_RANK_MATCH_EARLY_COUNTS_ANY,
+        cub::BLOCK_SCAN_RAKING_MEMOIZE,
+        1,
+        TUNE_RADIX_BITS};
+    }();
+
+    // These kernels are launched once, no point in tuning at the moment
+    const auto histogram = cub::RadixSortHistogramPolicy{
+      128, 16, cub::detail::radix_sort::__scale_num_parts(1, sizeof(KeyT)), TUNE_RADIX_BITS};
+    const auto exclusive_sum = cub::RadixSortExclusiveSumPolicy{256, TUNE_RADIX_BITS};
+
+    const auto scan = [] {
+      const auto scaled = cub::detail::scale_mem_bound(512, 23, sizeof(OffsetT));
+      return scan{scaled.threads_per_block,
+                  scaled.items_per_thread,
+                  cub::BLOCK_LOAD_WARP_TRANSPOSE,
+                  cub::LOAD_DEFAULT,
+                  cub::BLOCK_STORE_WARP_TRANSPOSE,
+                  cub::BLOCK_SCAN_RAKING_MEMOIZE};
+    }();
+
+    // No point in tuning
+    const int single_tile_radix_bits = (sizeof(KeyT) > 1) ? 6 : 5;
+
+    // No point in tuning single-tile policy
+    const auto single_tile = [] {
+      const auto scaled = cub::detail::scale_reg_bound(256, 19, sizeof(DominantT));
+      return cub::RadixSortDownsweepPolicy{
+        scaled.threads_per_block,
+        scaled.items_per_thread,
+        cub::BLOCK_LOAD_DIRECT,
+        cub::LOAD_LDG,
+        cub::RADIX_RANK_MEMOIZE,
+        cub::BLOCK_SCAN_WARP_SCANS,
+        single_tile_radix_bits,
+      };
+    }();
+
+    return cub::RadixSortPolicy{
+      cub::RadixSortAlgorithm::onesweep,
+      histogram,
+      exclusive_sum,
+      onesweep,
+      scan,
+      /* downsweep */ {},
+      /* alt_downsweep */ {},
+      /* upsweep */ {},
+      /* alt_upsweep */ {},
+      single_tile};
+  }
+};
+
+template <typename KeyT, typename ValueT, typename OffsetT, cub::SortOrder SortOrder>
+constexpr std::size_t max_onesweep_temp_storage_size()
+{
+  using portion_offset = int;
+
+  constexpr auto active_policy = policy_selector<KeyT, ValueT, OffsetT>{}(cuda::compute_capability{});
+
+  constexpr auto onesweep = active_policy.onesweep;
+  using onesweep_policy_t = cub::detail::agent_radix_sort_onesweep_policy<
+    0,
+    0,
+    void,
+    onesweep.rank_private_partitions,
+    onesweep.rank_algorithm,
+    onesweep.scan_algorithm,
+    onesweep.store_algorithm,
+    onesweep.radix_bits,
+    cub::NoScaling<onesweep.threads_per_block, onesweep.items_per_thread>>;
+
+  using agent_radix_sort_onesweep_t =
+    cub::AgentRadixSortOnesweep<onesweep_policy_t, SortOrder, KeyT, ValueT, OffsetT, portion_offset>;
+
+  constexpr auto histogram = active_policy.histogram;
+  using histogram_policy_t = cub::detail::agent_radix_sort_histogram_policy<
+    histogram.threads_per_block,
+    histogram.items_per_thread,
+    histogram.private_partitions,
+    void,
+    histogram.radix_bits>;
+  using hist_agent = cub::AgentRadixSortHistogram<histogram_policy_t, SortOrder, KeyT, OffsetT>;
+
+  return cuda::std::max(sizeof(typename agent_radix_sort_onesweep_t::TempStorage),
+                        sizeof(typename hist_agent::TempStorage));
+}
+
+template <typename KeyT, typename ValueT, typename OffsetT, cub::SortOrder SortOrder>
+constexpr std::size_t max_temp_storage_size()
+{
+  using offset_t               = cub::detail::choose_offset_t<OffsetT>;
+  constexpr auto active_policy = policy_selector<KeyT, ValueT, offset_t>{}(cuda::compute_capability{});
+  static_assert(active_policy.algorithm == cub::RadixSortAlgorithm::onesweep);
+  return max_onesweep_temp_storage_size<KeyT, ValueT, offset_t, SortOrder>();
+}
+
+template <typename KeyT, typename ValueT, typename OffsetT, cub::SortOrder SortOrder>
+constexpr bool fits_in_default_shared_memory()
+{
+  return max_temp_storage_size<KeyT, ValueT, OffsetT, SortOrder>() < cub::detail::max_smem_per_block;
+}
+#else // TUNE_BASE
+template <typename, typename, typename, auto>
+constexpr bool fits_in_default_shared_memory()
+{
+  return true;
+}
+#endif // TUNE_BASE

cccl_upstream/cub/benchmarks/bench/reduce/arg_extrema.cu

@@ -0,0 +1,95 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include <cub/device/device_reduce.cuh>
+#include <cub/device/dispatch/tuning/tuning_reduce.cuh>
+
+#include <cuda/std/type_traits>
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_ITEMS_PER_VEC_LOAD_POW2 ipv 1:2:1
+
+#if !TUNE_BASE
+struct tuned_policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ReducePolicy
+  {
+    cub::ReducePassPolicy rp{
+      TUNE_THREADS_PER_BLOCK,
+      TUNE_ITEMS_PER_THREAD,
+      1 << TUNE_ITEMS_PER_VEC_LOAD_POW2,
+      cub::BLOCK_REDUCE_WARP_REDUCTIONS,
+      cub::LOAD_DEFAULT};
+    return {rp, rp};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T, typename OpT>
+void arg_reduce(nvbench::state& state, nvbench::type_list<T, OpT>)
+{
+  // Offset type used to index within the total input in the range [d_in, d_in + num_items)
+  using offset_t = cuda::std::int64_t;
+
+  // Retrieve axis parameters
+  const auto elements         = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<offset_t> out_index(1);
+  thrust::device_vector<T> out_extremum(1);
+
+  const T* d_in         = thrust::raw_pointer_cast(in.data());
+  offset_t* d_out_index = thrust::raw_pointer_cast(out_index.data());
+  T* d_out_extremum     = thrust::raw_pointer_cast(out_extremum.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<offset_t>(1);
+  state.add_global_memory_writes<T>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(tuned_policy_selector{})
+#endif // !TUNE_BASE
+    );
+    if constexpr (cuda::std::is_same_v<OpT, cub::detail::arg_min>)
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceReduce::ArgMin,
+        "ArgMin failed",
+        d_in,
+        d_out_extremum,
+        d_out_index,
+        static_cast<offset_t>(elements),
+        cuda::std::less{},
+        env);
+    }
+    else
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceReduce::ArgMax,
+        "ArgMax failed",
+        d_in,
+        d_out_extremum,
+        d_out_index,
+        static_cast<offset_t>(elements),
+        cuda::std::less{},
+        env);
+    }
+  });
+}
+
+using op_types = nvbench::type_list<cub::detail::arg_min, cub::detail::arg_max>;
+
+NVBENCH_BENCH_TYPES(arg_reduce, NVBENCH_TYPE_AXES(fundamental_types, op_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "Operation{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/reduce/base.cuh

@@ -0,0 +1,69 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#pragma once
+
+#include <cub/device/device_reduce.cuh>
+
+#include <nvbench_helper.cuh>
+
+#if !TUNE_BASE
+template <typename AccumT>
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ReducePolicy
+  {
+    const auto [items, threads] =
+      cub::detail::scale_mem_bound(TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, int{sizeof(AccumT)});
+    const auto policy = cub::ReducePassPolicy{
+      threads, items, 1 << TUNE_ITEMS_PER_VEC_LOAD_POW2, cub::BLOCK_REDUCE_WARP_REDUCTIONS, cub::LOAD_DEFAULT};
+    return {policy, policy};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T, typename OffsetT>
+void reduce(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using init_value_t = T;
+
+  // Retrieve axis parameters
+  const auto elements = state.get_int64("Elements{io}");
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<T> out(1);
+
+  auto d_in  = thrust::raw_pointer_cast(in.data());
+  auto d_out = thrust::raw_pointer_cast(out.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<cuda::std::__accumulator_t<op_t, T, init_value_t>>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceReduce::Reduce,
+      "Reduce failed",
+      d_in,
+      d_out,
+      static_cast<OffsetT>(elements),
+      op_t{},
+      init_value_t{},
+      env);
+  });
+}
+
+NVBENCH_BENCH_TYPES(reduce, NVBENCH_TYPE_AXES(value_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/reduce/by_key.cu

@@ -0,0 +1,132 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_reduce.cuh>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+struct bench_reduce_by_key_policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ReduceByKeyPolicy
+  {
+    return {
+      cub::ReduceByKeyAlgorithm::lookback,
+      {
+        TUNE_THREADS,
+        TUNE_ITEMS,
+        TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE,
+        TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : cub::LOAD_CA,
+        cub::BLOCK_SCAN_WARP_SCANS,
+        lookback_delay_policy,
+      },
+    };
+  }
+};
+#endif // !TUNE_BASE
+
+template <class KeyT, class ValueT, class OffsetT>
+static void reduce_by_key(nvbench::state& state, nvbench::type_list<KeyT, ValueT, OffsetT>)
+{
+  using reduction_op_t = ::cuda::std::plus<>;
+
+  const auto elements                    = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  constexpr std::size_t min_segment_size = 1;
+  const std::size_t max_segment_size     = static_cast<std::size_t>(state.get_int64("MaxSegSize"));
+
+  thrust::device_vector<OffsetT> num_runs_out(1);
+  thrust::device_vector<ValueT> in_vals(elements);
+  thrust::device_vector<ValueT> out_vals(elements);
+  thrust::device_vector<KeyT> out_keys(elements);
+  thrust::device_vector<KeyT> in_keys = generate.uniform.key_segments(elements, min_segment_size, max_segment_size);
+
+  const KeyT* d_in_keys   = thrust::raw_pointer_cast(in_keys.data());
+  KeyT* d_out_keys        = thrust::raw_pointer_cast(out_keys.data());
+  const ValueT* d_in_vals = thrust::raw_pointer_cast(in_vals.data());
+  ValueT* d_out_vals      = thrust::raw_pointer_cast(out_vals.data());
+  OffsetT* d_num_runs_out = thrust::raw_pointer_cast(num_runs_out.data());
+
+  caching_allocator_t alloc;
+
+  // Run once to get the number of runs for reporting
+  _CCCL_TRY_CUDA_API(
+    cub::DeviceReduce::ReduceByKey,
+    "ReduceByKey failed",
+    d_in_keys,
+    d_out_keys,
+    d_in_vals,
+    d_out_vals,
+    d_num_runs_out,
+    reduction_op_t{},
+    static_cast<OffsetT>(elements),
+    alloc);
+  cudaDeviceSynchronize();
+  const OffsetT num_runs = num_runs_out[0];
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<ValueT>(num_runs);
+  state.add_global_memory_writes<KeyT>(num_runs);
+  state.add_global_memory_writes<OffsetT>(1);
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_reduce_by_key_policy_selector{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceReduce::ReduceByKey,
+      "ReduceByKey failed",
+      d_in_keys,
+      d_out_keys,
+      d_in_vals,
+      d_out_vals,
+      d_num_runs_out,
+      reduction_op_t{},
+      static_cast<OffsetT>(elements),
+      env);
+  });
+}
+
+using some_offset_types = nvbench::type_list<nvbench::int32_t>;
+
+#ifdef TUNE_KeyT
+using key_types = nvbench::type_list<TUNE_KeyT>;
+#else // !defined(TUNE_KeyT)
+using key_types =
+  nvbench::type_list<int8_t,
+                     int16_t,
+                     int32_t,
+                     int64_t
+#  if _CCCL_HAS_INT128()
+                     ,
+                     int128_t
+#  endif
+                     >;
+#endif // TUNE_KeyT
+
+#ifdef TUNE_ValueT
+using value_types = nvbench::type_list<TUNE_ValueT>;
+#else // !defined(TUNE_ValueT)
+using value_types = all_types;
+#endif // TUNE_ValueT
+
+NVBENCH_BENCH_TYPES(reduce_by_key, NVBENCH_TYPE_AXES(key_types, value_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegSize", {1, 4, 8});

cccl_upstream/cub/benchmarks/bench/reduce/custom.cu

@@ -0,0 +1,14 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+// This benchmark uses a custom reduction operation, max_t, which is not known to CUB, so no operator specific
+// optimizations (e.g. using redux or DPX instructions) are performed. This benchmark covers the unoptimized code path.
+
+// Because CUB cannot detect this operator, we cannot add any tunings based on the results of this benchmark. Its main
+// use is to detect regressions.
+
+#include <nvbench_helper.cuh>
+
+using value_types = all_types;
+using op_t        = max_t;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/reduce/deferred_deterministic.cu

@@ -0,0 +1,93 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/device/device_reduce.cuh>
+
+#include <thrust/detail/raw_pointer_cast.h>
+#include <thrust/device_vector.h>
+
+#include <cuda/argument>
+#include <cuda/execution.determinism.h>
+#include <cuda/execution.require.h>
+#include <cuda/std/functional>
+#include <cuda/std/utility>
+
+#include <nvbench_helper.cuh>
+
+#include <nvbench/range.cuh>
+#include <nvbench/types.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 3:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+
+#if !TUNE_BASE
+struct policy_selector_t
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ReducePolicy
+  {
+    const auto p = cub::ReducePassPolicy{
+      TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, 1, cub::BLOCK_REDUCE_RAKING, cub::LOAD_DEFAULT};
+    return {p, p};
+  }
+};
+#endif // !TUNE_BASE
+
+template <class T, class OffsetT>
+void deterministic_sum(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+try
+{
+  using init_value_t = T;
+
+  if (!cuda::std::in_range<OffsetT>(state.get_int64("Elements{io}")))
+  {
+    state.skip("Skipping: Elements{io} is not representable by OffsetT.");
+    return;
+  }
+  const auto elements = static_cast<OffsetT>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<T> out(1, thrust::no_init);
+  thrust::device_vector<OffsetT> device_num_items{elements};
+
+  auto d_in        = thrust::raw_pointer_cast(in.data());
+  auto d_out       = thrust::raw_pointer_cast(out.data());
+  auto d_num_items = thrust::raw_pointer_cast(device_num_items.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch,
+      cuda::execution::require(cuda::execution::determinism::gpu_to_gpu)
+#if !TUNE_BASE
+        ,
+      cuda::execution::tune(policy_selector_t{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceReduce::Reduce,
+      "Reduce failed",
+      d_in,
+      d_out,
+      cuda::args::deferred{d_num_items},
+      cuda::std::plus<>{},
+      init_value_t{},
+      env);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+using types = nvbench::type_list<float, double>;
+NVBENCH_BENCH_TYPES(deterministic_sum, NVBENCH_TYPE_AXES(types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  // 2^32 exceeds INT32_MAX to cover the code paths for problem sizes that exceed a single 32-bit chunk
+  .add_int64_power_of_two_axis("Elements{io}", {16, 20, 24, 28, 32});

cccl_upstream/cub/benchmarks/bench/reduce/deferred_nondeterministic.cu

@@ -0,0 +1,98 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/device/device_reduce.cuh>
+
+#include <thrust/detail/raw_pointer_cast.h>
+#include <thrust/device_vector.h>
+
+#include <cuda/argument>
+#include <cuda/execution.determinism.h>
+#include <cuda/execution.require.h>
+#include <cuda/std/functional>
+
+#include <cstddef>
+
+#include <nvbench_helper.cuh>
+
+#include <nvbench/range.cuh>
+#include <nvbench/types.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 3:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_ITEMS_PER_VEC_LOAD_POW2 ipv 1:2:1
+
+#if !TUNE_BASE
+template <typename AccumT>
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ReducePolicy
+  {
+    const auto [items, threads] =
+      cub::detail::scale_mem_bound(TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, int{sizeof(AccumT)});
+    const auto policy = cub::ReducePassPolicy{
+      threads,
+      items,
+      1 << TUNE_ITEMS_PER_VEC_LOAD_POW2,
+      cub::BLOCK_REDUCE_WARP_REDUCTIONS_NONDETERMINISTIC,
+      cub::LOAD_DEFAULT};
+    return {policy, {}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T, typename OffsetT>
+void nondeterministic_sum(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using op_t         = cuda::std::plus<>;
+  using init_value_t = T;
+
+  // Retrieve axis parameters
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<T> out(1, thrust::no_init);
+  thrust::device_vector<OffsetT> device_num_items(1, static_cast<OffsetT>(elements));
+
+  auto d_in        = thrust::raw_pointer_cast(in.data());
+  auto d_out       = thrust::raw_pointer_cast(out.data());
+  auto d_num_items = thrust::raw_pointer_cast(device_num_items.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch,
+      cuda::execution::require(cuda::execution::determinism::not_guaranteed)
+#if !TUNE_BASE
+        ,
+      cuda::execution::tune(policy_selector<cuda::std::__accumulator_t<op_t, T, init_value_t>>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceReduce::Reduce,
+      "Reduce failed",
+      d_in,
+      d_out,
+      cuda::args::deferred{d_num_items},
+      op_t{},
+      init_value_t{},
+      env);
+  });
+}
+
+#ifdef TUNE_T
+using value_types = nvbench::type_list<TUNE_T>;
+#else
+using value_types = nvbench::type_list<int32_t, int64_t, float, double>;
+#endif
+
+NVBENCH_BENCH_TYPES(nondeterministic_sum, NVBENCH_TYPE_AXES(value_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/reduce/deferred_sum.cu

@@ -0,0 +1,86 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/device/device_reduce.cuh>
+
+#include <thrust/detail/raw_pointer_cast.h>
+#include <thrust/device_vector.h>
+
+#include <cuda/argument>
+#include <cuda/std/functional>
+
+#include <nvbench_helper.cuh>
+
+#include <nvbench/range.cuh>
+#include <nvbench/types.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_ITEMS_PER_VEC_LOAD_POW2 ipv 1:2:1
+
+#if !TUNE_BASE
+template <typename AccumT>
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ReducePolicy
+  {
+    const auto [items, threads] =
+      cub::detail::scale_mem_bound(TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, int{sizeof(AccumT)});
+    const auto policy = cub::ReducePassPolicy{
+      threads, items, 1 << TUNE_ITEMS_PER_VEC_LOAD_POW2, cub::BLOCK_REDUCE_WARP_REDUCTIONS, cub::LOAD_DEFAULT};
+    return {policy, policy};
+  }
+};
+#endif // !TUNE_BASE
+
+using op_t = cuda::std::plus<>;
+
+template <typename T, typename OffsetT>
+void reduce(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using init_value_t = T;
+
+  // Retrieve axis parameters
+  const auto elements = state.get_int64("Elements{io}");
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<T> out(1, thrust::default_init);
+  thrust::device_vector<OffsetT> device_num_items(1, static_cast<OffsetT>(elements));
+
+  auto d_in        = thrust::raw_pointer_cast(in.data());
+  auto d_out       = thrust::raw_pointer_cast(out.data());
+  auto d_num_items = thrust::raw_pointer_cast(device_num_items.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<cuda::std::__accumulator_t<op_t, T, init_value_t>>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceReduce::Reduce,
+      "Reduce failed",
+      d_in,
+      d_out,
+      cuda::args::deferred{d_num_items},
+      op_t{},
+      init_value_t{},
+      env);
+  });
+}
+
+using value_types = all_types;
+
+NVBENCH_BENCH_TYPES(reduce, NVBENCH_TYPE_AXES(value_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/reduce/deterministic.cu

@@ -0,0 +1,77 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_reduce.cuh>
+
+#include <cuda/execution.determinism.h>
+#include <cuda/execution.require.h>
+#include <cuda/std/utility>
+
+#include <nvbench_helper.cuh>
+
+#include <nvbench/range.cuh>
+#include <nvbench/types.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 3:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+
+#if !TUNE_BASE
+struct policy_selector_t
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ReducePolicy
+  {
+    const auto p = cub::ReducePassPolicy{
+      TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, 1, cub::BLOCK_REDUCE_RAKING, cub::LOAD_DEFAULT};
+    return {p, p};
+  }
+};
+#endif // !TUNE_BASE
+
+template <class T, class OffsetT>
+void deterministic_sum(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+try
+{
+  using init_value_t = T;
+
+  if (!cuda::std::in_range<OffsetT>(state.get_int64("Elements{io}")))
+  {
+    state.skip("Skipping: Elements{io} is not representable by OffsetT.");
+    return;
+  }
+  const auto elements = static_cast<OffsetT>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<T> out(1);
+
+  const T* d_in = thrust::raw_pointer_cast(in.data());
+  T* d_out      = thrust::raw_pointer_cast(out.data());
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(out.size());
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch,
+      cuda::execution::require(cuda::execution::determinism::gpu_to_gpu)
+#if !TUNE_BASE
+        ,
+      cuda::execution::tune(policy_selector_t{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceReduce::Reduce, "Reduce failed", d_in, d_out, elements, cuda::std::plus<>{}, init_value_t{}, env);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+using types = nvbench::type_list<float, double>;
+NVBENCH_BENCH_TYPES(deterministic_sum, NVBENCH_TYPE_AXES(types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  // 2^32 exceeds INT32_MAX to cover the code paths for problem sizes that exceed a single 32-bit chunk
+  .add_int64_power_of_two_axis("Elements{io}", {16, 20, 24, 28, 32});

cccl_upstream/cub/benchmarks/bench/reduce/min.cu

@@ -0,0 +1,36 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+// This benchmark is intended to cover DPX instructions on Hopper+ architectures. It specifically uses cuda::minimum<>
+// instead of a user-defined operator, which CUB recognizes to select an optimized code path.
+
+// Tuning parameters found for ::cuda::minimum<> apply equally for ::cuda::maximum<>
+// Tuning parameters found for signed integer types apply equally for unsigned integer types
+// TODO(bgruber): do tuning parameters found for int16_t apply equally for __half or __nv_bfloat16 on SM90+?
+
+#include <cuda/functional>
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_ITEMS_PER_VEC_LOAD_POW2 ipv 1:2:1
+
+// __half and __nv_bfloat16 are added for full (non-tuning) runs; CUB has fast paths for them (see #9587).
+#ifdef TUNE_T
+using value_types = nvbench::type_list<TUNE_T>;
+#else
+using value_types =
+  push_back_t<fundamental_types
+#  if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
+              ,
+              __half
+#  endif
+#  if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
+              ,
+              __nv_bfloat16
+#  endif
+              >;
+#endif
+using op_t = ::cuda::minimum<>;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/reduce/nondeterministic.cu

@@ -0,0 +1,89 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include <cub/device/device_reduce.cuh>
+
+#include <cuda/execution.determinism.h>
+#include <cuda/execution.require.h>
+
+#include <nvbench_helper.cuh>
+
+#include <nvbench/range.cuh>
+#include <nvbench/types.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 3:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_ITEMS_PER_VEC_LOAD_POW2 ipv 1:2:1
+
+#if !TUNE_BASE
+template <typename AccumT>
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ReducePolicy
+  {
+    const auto [items, threads] =
+      cub::detail::scale_mem_bound(TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, int{sizeof(AccumT)});
+    const auto policy = cub::ReducePassPolicy{
+      threads,
+      items,
+      1 << TUNE_ITEMS_PER_VEC_LOAD_POW2,
+      cub::BLOCK_REDUCE_WARP_REDUCTIONS_NONDETERMINISTIC,
+      cub::LOAD_DEFAULT};
+    return {policy, {}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T, typename OffsetT>
+void nondeterministic_sum(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using op_t         = cuda::std::plus<>;
+  using init_value_t = T;
+
+  // Retrieve axis parameters
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<T> out(1);
+
+  auto d_in  = thrust::raw_pointer_cast(in.data());
+  auto d_out = thrust::raw_pointer_cast(out.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch,
+      cuda::execution::require(cuda::execution::determinism::not_guaranteed)
+#if !TUNE_BASE
+        ,
+      cuda::execution::tune(policy_selector<cuda::std::__accumulator_t<op_t, T, init_value_t>>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceReduce::Reduce,
+      "Reduce failed",
+      d_in,
+      d_out,
+      static_cast<OffsetT>(elements),
+      op_t{},
+      init_value_t{},
+      env);
+  });
+}
+
+#ifdef TUNE_T
+using value_types = nvbench::type_list<TUNE_T>;
+#else
+using value_types = nvbench::type_list<int32_t, int64_t, float, double>;
+#endif
+
+NVBENCH_BENCH_TYPES(nondeterministic_sum, NVBENCH_TYPE_AXES(value_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/reduce/sum.cu

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+// This benchmark is intended to cover redux instructions on Ampere+ architectures. It specifically uses
+// cuda::std::plus<> instead of a user-defined operator, which CUB recognizes to select an optimized code path.
+
+// Tuning parameters found for signed integer types apply equally for unsigned integer types
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_ITEMS_PER_VEC_LOAD_POW2 ipv 1:2:1
+
+// __half and __nv_bfloat16 are added for full (non-tuning) runs; CUB has fast paths for them (see #9587).
+#ifdef TUNE_T
+using value_types = nvbench::type_list<TUNE_T>;
+#else
+using value_types =
+  push_back_t<all_types
+#  if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
+              ,
+              __half
+#  endif
+#  if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
+              ,
+              __nv_bfloat16
+#  endif
+              >;
+#endif
+using op_t = ::cuda::std::plus<>;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/reduce/warp_reduce_base.cuh

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#pragma once
+
+#include <cub/config.cuh>
+
+#include <cuda_runtime_api.h>
+#include <device_side_benchmark.cuh>
+#include <nvbench_helper.cuh>
+
+struct benchmark_op_t
+{
+  template <typename T>
+  __device__ __forceinline__ T operator()(T thread_data) const
+  {
+    using WarpReduce  = cub::WarpReduce<T>;
+    using TempStorage = typename WarpReduce::TempStorage;
+    __shared__ TempStorage temp_storage[32];
+    auto warp_id = threadIdx.x / 32;
+    return WarpReduce{temp_storage[warp_id]}.Reduce(thread_data, op_t{});
+  }
+};
+
+template <typename T>
+void warp_reduce(nvbench::state& state, nvbench::type_list<T>)
+{
+  constexpr int block_size    = 256;
+  constexpr int unroll_factor = 128; // compromise between compile time and noise
+  const auto& kernel          = benchmark_kernel<block_size, unroll_factor, benchmark_op_t, T>;
+  const int num_SMs     = state.get_device().value().get_number_of_sms(); // NOLINT(bugprone-unchecked-optional-access)
+  const int device      = state.get_device().value().get_id(); // NOLINT(bugprone-unchecked-optional-access)
+  int max_blocks_per_SM = 0;
+  NVBENCH_CUDA_CALL_NOEXCEPT(cudaOccupancyMaxActiveBlocksPerMultiprocessor(&max_blocks_per_SM, kernel, block_size, 0));
+  const int grid_size = max_blocks_per_SM * num_SMs;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch&) {
+    kernel<<<grid_size, block_size>>>(benchmark_op_t{});
+  });
+}
+
+NVBENCH_BENCH_TYPES(warp_reduce, NVBENCH_TYPE_AXES(value_types)).set_name("base").set_type_axes_names({"T{ct}"});

cccl_upstream/cub/benchmarks/bench/reduce/warp_reduce_min.cu

@@ -0,0 +1,30 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <nvbench_helper.cuh>
+
+// complex types cannot be compared with operator<
+using value_types = nvbench::type_list<
+  int8_t,
+  int16_t,
+  int32_t,
+  int64_t,
+#if _CCCL_HAS_INT128()
+  int128_t,
+#endif
+#if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
+  __half,
+#endif
+#if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
+  __nv_bfloat16,
+#endif
+  float,
+  double
+#if _CCCL_HAS_FLOAT128()
+  ,
+  __float128
+#endif
+  >;
+
+using op_t = ::cuda::minimum<>;
+#include "warp_reduce_base.cuh"

cccl_upstream/cub/benchmarks/bench/reduce/warp_reduce_sum.cu

@@ -0,0 +1,35 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <nvbench_helper.cuh>
+
+using value_types = nvbench::type_list<
+  int8_t,
+  int16_t,
+  int32_t,
+  int64_t,
+#if _CCCL_HAS_INT128()
+  int128_t,
+#endif
+#if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
+  __half,
+#endif
+#if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
+  __nv_bfloat16,
+#endif
+  float,
+  double,
+#if _CCCL_HAS_FLOAT128()
+  __float128,
+#endif
+#if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
+  cuda::std::complex<__half>,
+#endif
+#if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
+  cuda::std::complex<__nv_bfloat16>,
+#endif
+  cuda::std::complex<float>,
+  cuda::std::complex<double>>;
+
+using op_t = ::cuda::std::plus<>;
+#include "warp_reduce_base.cuh"

cccl_upstream/cub/benchmarks/bench/run_length_encode/encode.cu

@@ -0,0 +1,96 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_run_length_encode.cuh>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+struct bench_encode_policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::RleEncodePolicy
+  {
+    return {
+      cub::RleAlgorithm::lookback,
+      {TUNE_THREADS,
+       TUNE_ITEMS,
+       TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE,
+       TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : cub::LOAD_CA,
+       cub::BLOCK_SCAN_WARP_SCANS,
+       lookback_delay_policy},
+    };
+  }
+};
+#endif // !TUNE_BASE
+
+//! @tparam RunLengthT Offset type large enough to represent the longest run in the sequence
+template <class T, class OffsetT, class RunLengthT>
+static void rle(nvbench::state& state, nvbench::type_list<T, OffsetT, RunLengthT>)
+{
+  // Offset type large enough to represent any offset into the input sequence and the total number of runs
+  using offset_t = cub::detail::choose_signed_offset_t<OffsetT>;
+
+  const auto elements                    = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  constexpr std::size_t min_segment_size = 1;
+  const std::size_t max_segment_size     = static_cast<std::size_t>(state.get_int64("MaxSegSize"));
+
+  thrust::device_vector<offset_t> num_runs_out(1);
+  thrust::device_vector<RunLengthT> out_counts(elements);
+  thrust::device_vector<T> out_keys(elements);
+  thrust::device_vector<T> in_keys = generate.uniform.key_segments(elements, min_segment_size, max_segment_size);
+
+  const T* d_in_keys       = thrust::raw_pointer_cast(in_keys.data());
+  T* d_out_keys            = thrust::raw_pointer_cast(out_keys.data());
+  RunLengthT* d_out_counts = thrust::raw_pointer_cast(out_counts.data());
+  offset_t* d_num_runs_out = thrust::raw_pointer_cast(num_runs_out.data());
+
+  // Run once to get num_runs for memory accounting
+  (void) cub::DeviceRunLengthEncode::Encode(
+    d_in_keys, d_out_keys, d_out_counts, d_num_runs_out, static_cast<OffsetT>(elements));
+  cudaDeviceSynchronize();
+  const offset_t num_runs = num_runs_out[0];
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(num_runs);
+  state.add_global_memory_writes<RunLengthT>(num_runs);
+  state.add_global_memory_writes<offset_t>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_encode_policy_selector{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceRunLengthEncode::Encode,
+      "Encode failed",
+      d_in_keys,
+      d_out_keys,
+      d_out_counts,
+      d_num_runs_out,
+      static_cast<OffsetT>(elements),
+      env);
+  });
+}
+
+using run_length_types = nvbench::type_list<nvbench::int32_t, nvbench::int64_t>;
+
+NVBENCH_BENCH_TYPES(rle, NVBENCH_TYPE_AXES(all_types, offset_types, run_length_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}", "RunLengthT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegSize", {1, 4, 8});

cccl_upstream/cub/benchmarks/bench/run_length_encode/non_trivial_runs.cu

@@ -0,0 +1,113 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_run_length_encode.cuh>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_TIME_SLICING ts 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+struct bench_rle_policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const
+    -> cub::RleNonTrivialRunsPolicy
+  {
+    return {
+      cub::RleNonTrivialRunsAlgorithm::lookback,
+      {
+        TUNE_THREADS,
+        TUNE_ITEMS,
+        TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE,
+        TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : cub::LOAD_CA,
+        static_cast<bool>(TUNE_TIME_SLICING),
+        cub::BLOCK_SCAN_WARP_SCANS,
+        lookback_delay_policy,
+      },
+    };
+  }
+};
+#endif // !TUNE_BASE
+
+template <class T, class OffsetT, class RunLengthT>
+static void rle(nvbench::state& state, nvbench::type_list<T, OffsetT, RunLengthT>)
+{
+  using offset_t = cub::detail::choose_signed_offset_t<OffsetT>;
+
+  const auto elements                    = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  constexpr std::size_t min_segment_size = 1;
+  const std::size_t max_segment_size     = static_cast<std::size_t>(state.get_int64("MaxSegSize"));
+
+  thrust::device_vector<offset_t> num_runs_out(1);
+  thrust::device_vector<offset_t> out_offsets(elements);
+  thrust::device_vector<RunLengthT> out_lengths(elements);
+  thrust::device_vector<T> in_keys = generate.uniform.key_segments(elements, min_segment_size, max_segment_size);
+
+  const T* d_in_keys        = thrust::raw_pointer_cast(in_keys.data());
+  offset_t* d_out_offsets   = thrust::raw_pointer_cast(out_offsets.data());
+  RunLengthT* d_out_lengths = thrust::raw_pointer_cast(out_lengths.data());
+  offset_t* d_num_runs_out  = thrust::raw_pointer_cast(num_runs_out.data());
+
+  {
+    // Run once to get num_runs for memory accounting
+    auto memory_env = cuda::std::execution::env{
+#if !TUNE_BASE
+      cuda::execution::tune(bench_rle_policy_selector{})
+#endif // !TUNE_BASE
+    };
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceRunLengthEncode::NonTrivialRuns,
+      "NonTrivialRuns failed",
+      d_in_keys,
+      d_out_offsets,
+      d_out_lengths,
+      d_num_runs_out,
+      static_cast<OffsetT>(elements),
+      memory_env);
+    cudaDeviceSynchronize();
+  }
+  const OffsetT num_runs = num_runs_out[0];
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<OffsetT>(num_runs);
+  state.add_global_memory_writes<OffsetT>(num_runs);
+  state.add_global_memory_writes<OffsetT>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_rle_policy_selector{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceRunLengthEncode::NonTrivialRuns,
+      "NonTrivialRuns failed",
+      d_in_keys,
+      d_out_offsets,
+      d_out_lengths,
+      d_num_runs_out,
+      static_cast<OffsetT>(elements),
+      env);
+  });
+}
+
+using run_length_types = nvbench::type_list<nvbench::int32_t, nvbench::int64_t>;
+
+NVBENCH_BENCH_TYPES(rle, NVBENCH_TYPE_AXES(all_types, offset_types, run_length_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}", "RunLengthT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegSize", {1, 4, 8});

cccl_upstream/cub/benchmarks/bench/scan/applications/P1/log-cdf-from-log-pdfs.cu

@@ -0,0 +1,170 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/device_scan.cuh>
+
+#include <thrust/device_vector.h>
+#include <thrust/host_vector.h>
+
+#include <cuda/std/cmath>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_DIRECT
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_WARP_TRANSPOSE
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+#endif // !TUNE_BASE
+
+#include "../../policy_selector.h"
+
+namespace impl
+{
+/*
+ * Given a sequence of logarithms of probability mass function values,
+ * compute sequence of logarithms of cumulative distribution function values.
+ *
+ *    log(CDF(n)) = log(\sum( PDF(k), 0 <=k <=n ))
+ *
+ * This is inclusive scan using logaddexp binary operator:
+ *    logaddexp( logpdf1, logpdf2 ) := log( exp(logpdf1) + exp(logpdf2) )
+ *         == max(logpdf1, logpdf2) + log( 1 + exp(-abs(logpdf1 - logpdf2)))
+ *
+ * The last reformulation allows avoid numerical accuracy issues
+ * caused by underflows.
+ *
+ */
+
+struct log_add_plus
+{
+  /* Operator is commutative and associative */
+  template <typename T>
+  T __host__ __device__ operator()(T v1, T v2)
+  {
+    T max12 = cuda::maximum{}(v1, v2);
+    T min12 = cuda::minimum{}(v1, v2);
+    T exp   = cuda::std::exp(min12 - max12);
+    return max12 + cuda::std::log1p(exp);
+  }
+};
+
+template <typename T>
+struct log_pdf_builder
+{
+  T mu;
+  T norm;
+  cuda::std::size_t n;
+
+  T __host__ __device__ operator()(cuda::std::size_t i) const
+  {
+    return -mu * static_cast<T>(n - i) + norm;
+  }
+};
+
+template <typename T>
+[[nodiscard]] bool validate(const thrust::device_vector<T>& output, cudaStream_t stream)
+{
+  cudaStreamSynchronize(stream);
+
+  thrust::host_vector<T> h_output(output);
+  auto elements = h_output.size();
+  // test is designed so that last element of prefix scan sequence should be close to log(1.0) == 0.0
+  bool check = cuda::std::abs(h_output[elements - 1])
+             < cuda::std::sqrt(static_cast<T>(1 + elements)) * cuda::std::numeric_limits<T>::epsilon();
+
+  return check;
+}
+}; // namespace impl
+
+template <typename FloatingPointT, typename OffsetT>
+static void inclusive_scan(nvbench::state& state, nvbench::type_list<FloatingPointT, OffsetT>)
+{
+  static_assert(cuda::std::is_floating_point_v<FloatingPointT>);
+
+  using value_t                  = FloatingPointT;
+  using input_t                  = const value_t*;
+  using output_t                 = value_t*;
+  using op_t                     = impl::log_add_plus;
+  using accum_t [[maybe_unused]] = value_t;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  auto mu             = static_cast<value_t>(state.get_float64("Mu{io}"));
+
+  auto norm = cuda::std::log1p(-cuda::std::exp(-mu)) - cuda::std::log1p(-cuda::std::exp(-mu * elements));
+
+  thrust::device_vector<value_t> input(elements, thrust::no_init);
+
+  cudaStream_t bench_stream = state.get_cuda_stream();
+
+  auto naturals_it = cuda::counting_iterator(cuda::std::size_t{0});
+  cub::DeviceTransform::Transform(
+    cuda::std::make_tuple(naturals_it),
+    input.begin(),
+    elements,
+    impl::log_pdf_builder<value_t>{mu, norm, elements},
+    bench_stream);
+
+  thrust::device_vector<value_t> output(elements, thrust::no_init);
+
+  input_t d_input   = thrust::raw_pointer_cast(input.data());
+  output_t d_output = thrust::raw_pointer_cast(output.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<value_t>(elements, "Size");
+  state.add_global_memory_writes<value_t>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::InclusiveScan,
+      "InclusiveScan failed",
+      d_input,
+      d_output,
+      op_t{},
+      static_cast<OffsetT>(input.size()),
+      env);
+  });
+
+  // for validation, use
+  // assert(impl::validate(output, bench_stream));
+}
+
+#ifdef TUNE_T
+using fp_types = nvbench::type_list<TUNE_T>;
+#else
+using fp_types = nvbench::type_list<float, double>;
+#endif
+
+NVBENCH_BENCH_TYPES(inclusive_scan, NVBENCH_TYPE_AXES(fp_types, offset_types))
+  .set_name("app-logcdf-from-logpdf")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_float64_axis("Mu{io}", {1e-4f});

cccl_upstream/cub/benchmarks/bench/scan/applications/P1/non-commutative-bicyclic-monoid.cu

@@ -0,0 +1,153 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/device_scan.cuh>
+#include <cub/device/device_transform.cuh>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_DIRECT
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_WARP_TRANSPOSE
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+#endif // !TUNE_BASE
+
+#include "../../policy_selector.h"
+
+namespace impl
+{
+/* Consider free monoid with two generators, ``q`` and ``p``, modulo defining relationship (``p * q == 1``).
+ * Elements of this algebra are ``q^m * p^n``, identified by a pair of integral exponents. The identity
+ * element is ``1 == q^0 * p^0``, which maps to pair of zeros ``e = (0, 0)``.
+ *
+ * The product is defined by concatenation:
+ *     q^m * p^n * q^r * p^s == q^m * p^{n-1} * p * q * q^{r-1} * p^s
+ *                           == q^m * p^{n-1} * q^{r-1} * p^s
+ *
+ * This reduction can be performed ``min(n, r)`` times resulting in
+ *
+ *     q^m * p^n * q^r * p^s == q^{m + r - min(n, r)} * p^{s + n - min(n, r)}
+ *
+ * Hence this is a monoid, known as bicyclic monoid.
+ * This operation of pairs of integers is associative (since concatenation is), but non-commutative.
+ *
+ *    Ref: https://en.wikipedia.org/wiki/Bicyclic_semigroup
+ *    Ref: https://en.wikipedia.org/wiki/Monoid
+ */
+
+template <typename UnsignedIntegralT>
+struct bicyclic_monoid_op
+{
+  static_assert(cuda::std::is_integral_v<UnsignedIntegralT>);
+  static_assert(cuda::std::is_unsigned_v<UnsignedIntegralT>);
+
+  using pair_t = cuda::std::pair<UnsignedIntegralT, UnsignedIntegralT>;
+  using min_t  = cuda::minimum<>;
+
+  // Operator is associative but non-commutative
+  pair_t __host__ __device__ operator()(pair_t v1, pair_t v2) const
+  {
+    auto [m, n] = v1;
+    auto [r, s] = v2;
+    auto min_nr = min_t{}(n, r);
+    return {m + r - min_nr, s + n - min_nr};
+  }
+};
+
+template <typename T>
+struct repack_pair
+{
+  cuda::std::pair<T, T> __host__ __device__ operator()(const T& v1, const T& v2) const
+  {
+    return {v1, v2};
+  };
+};
+}; // namespace impl
+
+template <typename T, typename OffsetT>
+static void inclusive_scan(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  static_assert(cuda::std::is_integral_v<T> && cuda::std::is_unsigned_v<T>, "Unsigned integral type should be used");
+  using pair_t                   = cuda::std::pair<T, T>;
+  using op_t                     = impl::bicyclic_monoid_op<T>;
+  using accum_t [[maybe_unused]] = pair_t;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<pair_t> output(elements);
+
+  thrust::device_vector<pair_t> input(elements);
+  {
+    thrust::device_vector<T> q_exponents = generate(elements);
+    thrust::device_vector<T> p_exponents = generate(elements);
+
+    impl::repack_pair<T> repack_op{};
+
+    cub::DeviceTransform::Transform(
+      cuda::std::tuple{q_exponents.begin(), p_exponents.begin()}, input.begin(), elements, repack_op);
+
+    // deallocate temporary arrays at the scope boundary
+  }
+
+  pair_t* d_input  = thrust::raw_pointer_cast(input.data());
+  pair_t* d_output = thrust::raw_pointer_cast(output.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<pair_t>(elements, "Size");
+  state.add_global_memory_writes<pair_t>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::InclusiveScan,
+      "InclusiveScan failed",
+      d_input,
+      d_output,
+      op_t{},
+      static_cast<OffsetT>(input.size()),
+      env);
+  });
+}
+
+#ifdef TUNE_T
+using uint_types = nvbench::type_list<TUNE_T>;
+#else
+#  if _CCCL_HAS_INT128()
+using uint_types = nvbench::type_list<cuda::std::uint32_t, cuda::std::uint64_t, uint128_t>;
+#  else
+using uint_types = nvbench::type_list<cuda::std::uint32_t, cuda::std::uint64_t>;
+#  endif
+#endif
+
+NVBENCH_BENCH_TYPES(inclusive_scan, NVBENCH_TYPE_AXES(uint_types, offset_types))
+  .set_name("app-bicyclic-monoid")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/scan/applications/P1/rabin-karp-second-fingerprinting.cu

@@ -0,0 +1,347 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/device_scan.cuh>
+
+#include <thrust/host_vector.h>
+
+#include <cuda/cmath>
+#include <cuda/std/limits>
+
+#include <iostream>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_DIRECT
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_WARP_TRANSPOSE
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+#endif // !TUNE_BASE
+
+#include "../../policy_selector.h"
+
+namespace impl
+{
+/* Denote epsilon, the identity element, be an empty sequence, and consider
+ * set of sequences of {0, 1} bits, with binary operation of concatenation.
+ *
+ * Define homomorphism K from the set of sequence to 2-by-2 integral matrices
+ * over cyclic ring Z_p for some prime p.
+ *
+ *    K( ''  ) = [[ 1, 0], [0, 1]]
+ *    K( '0' ) = [[1, 0], [1, 1]]
+ *    K( '1' ) = [[1, 1], [0, 1]]
+ *
+ *    K( concat(seq1, seq2) ) := matmul( K(seq1), K(seq2) ) in Z_p
+ *
+ * Given a sequence of unsigned integers, encoding bit sequences,
+ * we build transform iterator mapping integer to the matrix. Then
+ * call inclusive_scan with matrix multiply operator in Z_p
+ *
+ * Ref: https://doi.org/10.1147/rd.312.0249
+ */
+
+// Types associated with the cyclic ring
+using ZpT   = cuda::std::uint32_t;
+using WideT = cuda::std::uint64_t;
+
+using MatT = cuda::std::array<ZpT, 4>;
+
+inline ZpT __host__ __device__ Zp_mul(ZpT v1, ZpT v2, cuda::fast_mod_div<WideT> m_p)
+{
+  const auto w1 = static_cast<WideT>(v1);
+  const auto w2 = static_cast<WideT>(v2);
+  return static_cast<ZpT>((w1 * w2) % m_p);
+}
+
+inline ZpT __host__ __device__ Zp_add(ZpT v1, ZpT v2, cuda::fast_mod_div<WideT> m_p)
+{
+  const auto w1 = static_cast<WideT>(v1);
+  const auto w2 = static_cast<WideT>(v2);
+  return static_cast<ZpT>((w1 + w2) % m_p);
+}
+
+inline MatT __host__ __device__ Zp_matmul(MatT v1, MatT v2, cuda::fast_mod_div<WideT> m_p)
+{
+  ZpT _1_00_2_00 = Zp_mul(v1[0], v2[0], m_p);
+  ZpT _1_01_2_10 = Zp_mul(v1[1], v2[2], m_p);
+  ZpT _r_00      = Zp_add(_1_00_2_00, _1_01_2_10, m_p);
+
+  ZpT _1_00_2_01 = Zp_mul(v1[0], v2[1], m_p);
+  ZpT _1_01_2_11 = Zp_mul(v1[1], v2[3], m_p);
+  ZpT _r_01      = Zp_add(_1_00_2_01, _1_01_2_11, m_p);
+
+  ZpT _1_10_2_00 = Zp_mul(v1[2], v2[0], m_p);
+  ZpT _1_11_2_10 = Zp_mul(v1[3], v2[2], m_p);
+  ZpT _r_10      = Zp_add(_1_10_2_00, _1_11_2_10, m_p);
+
+  ZpT _1_10_2_01 = Zp_mul(v1[2], v2[1], m_p);
+  ZpT _1_11_2_11 = Zp_mul(v1[3], v2[3], m_p);
+  ZpT _r_11      = Zp_add(_1_10_2_01, _1_11_2_11, m_p);
+
+  return {_r_00, _r_01, _r_10, _r_11};
+}
+
+struct RabinKarpOp
+{
+  cuda::fast_mod_div<WideT> m_p;
+
+  __host__ __device__ RabinKarpOp(ZpT p)
+      : m_p(static_cast<WideT>(p))
+  {}
+
+  // scan operator: non-commutative and associative
+  MatT __host__ __device__ operator()(MatT v1, MatT v2) const
+  {
+    return Zp_matmul(v1, v2, m_p);
+  }
+};
+
+template <typename T>
+struct ChunkToMat
+{
+  static_assert(cuda::std::is_integral_v<T> && cuda::std::is_unsigned_v<T>, "Bit sequence should be represented");
+
+  cuda::fast_mod_div<WideT> m_p;
+
+  __host__ __device__ ChunkToMat(ZpT p)
+      : m_p(static_cast<WideT>(p))
+  {}
+
+  MatT __host__ __device__ operator()(const T& bits) const
+  {
+    static constexpr int n_bits = cuda::std::numeric_limits<T>::digits;
+    static_assert(n_bits >= 1, "Type must have non-zero bitwidth");
+
+    static constexpr MatT _0 = {ZpT{1}, ZpT{0}, ZpT{1}, ZpT{1}}; // [[1, 0], [1, 1]]
+    static constexpr MatT _1 = {ZpT{1}, ZpT{1}, ZpT{0}, ZpT{1}}; // [[1, 1], [0, 1]]
+
+    // initialize with identity matrix
+    MatT m  = (bits & 1) ? _1 : _0;
+    T _bits = bits >> 1;
+
+    // use of cuda::static_for here results in performance regression due to increased register pressure
+    for (int i = 1; i < n_bits; ++i)
+    {
+      (void) i;
+      m = Zp_matmul((_bits & 1) ? _1 : _0, m, m_p);
+      _bits >>= 1;
+    }
+
+    return m;
+  }
+};
+
+// Iterator that performs assignment at specific index only, discards otherwise
+//
+// This iterator allows tp use inclusive_scan to perform reduction with
+// non-commutative associative binary operator
+//
+template <typename OffsetT, typename Iter>
+struct write_at_specific_index_or_discard
+{
+private:
+  OffsetT m_index{};
+  OffsetT m_target_index;
+  Iter m_iter;
+
+  void __host__ __device__ set_index(OffsetT index)
+  {
+    m_index = index;
+  }
+
+public:
+  struct assign_proxy
+  {
+  private:
+    bool m_writable;
+    Iter m_iter;
+
+  public:
+    __host__ __device__ assign_proxy(bool writable, Iter iter)
+        : m_writable(writable)
+        , m_iter(iter)
+    {}
+
+    template <typename Tp>
+    constexpr assign_proxy& __host__ __device__ operator=(Tp&& v)
+    {
+      if (m_writable)
+      {
+        *m_iter = v;
+      }
+      return *this;
+    }
+  };
+
+  using iterator_concept  = cuda::std::random_access_iterator_tag;
+  using iterator_category = cuda::std::random_access_iterator_tag;
+  using value_type        = cuda::std::iter_value_t<Iter>;
+  using difference_type   = cuda::std::iter_difference_t<Iter>;
+  using pointer           = void;
+  using reference         = void;
+
+  write_at_specific_index_or_discard() = delete;
+  explicit __host__ __device__ write_at_specific_index_or_discard(OffsetT offset, Iter iter)
+      : m_target_index(offset)
+      , m_iter(iter)
+  {}
+
+  write_at_specific_index_or_discard(const write_at_specific_index_or_discard&)            = default;
+  write_at_specific_index_or_discard(write_at_specific_index_or_discard&&)                 = default;
+  write_at_specific_index_or_discard& operator=(const write_at_specific_index_or_discard&) = default;
+  write_at_specific_index_or_discard& operator=(write_at_specific_index_or_discard&&)      = default;
+
+  assign_proxy __host__ __device__ operator[](difference_type n)
+  {
+    return {(m_index + static_cast<OffsetT>(n)) == m_target_index, m_iter};
+  }
+
+  write_at_specific_index_or_discard __host__ __device__ operator+(difference_type n) const
+  {
+    auto r = write_at_specific_index_or_discard(m_target_index, m_iter);
+    r.set_index((m_index + static_cast<OffsetT>(n)));
+    return r;
+  }
+};
+
+template <typename InputT, typename OutputT>
+[[nodiscard]] bool validate(
+  const thrust::device_vector<InputT>& input, const thrust::device_vector<OutputT>& output, ZpT p, cudaStream_t stream)
+{
+  using accum_t = OutputT;
+  using input_t = InputT;
+
+  cudaStreamSynchronize(stream);
+
+  thrust::host_vector<accum_t> h_out(output);
+  thrust::host_vector<input_t> h_inp(input);
+
+  accum_t ref_mat = {1, 0, 0, 1};
+
+  static constexpr accum_t mat_0 = {1, 0, 1, 1}; // lower diagonal
+  static constexpr accum_t mat_1 = {1, 1, 0, 1}; // upper diagonal
+  cuda::fast_mod_div<impl::WideT> mod(p);
+  for (auto&& el : h_inp)
+  {
+    input_t v = el;
+
+    accum_t word_mat = {1, 0, 0, 1};
+    for (int i = 0; i < sizeof(input_t) * 8; ++i)
+    {
+      if (v & 1)
+      {
+        word_mat = impl::Zp_matmul(mat_1, word_mat, mod);
+      }
+      else
+      {
+        word_mat = impl::Zp_matmul(mat_0, word_mat, mod);
+      }
+      v >>= 1;
+    }
+
+    ref_mat = impl::Zp_matmul(ref_mat, word_mat, mod);
+  }
+
+  const accum_t& res = h_out[0];
+  if (ref_mat != res)
+  {
+    std::cout << "FAILED: ";
+    std::cout << "cub_computed([[" << res[0] << ", " << res[1] << "], [" << res[2] << ", " << res[3] << "]]) != ";
+    std::cout
+      << "reference([[" << ref_mat[0] << ", " << ref_mat[1] << "], [" << ref_mat[2] << ", " << ref_mat[3] << "]])\n";
+    return false;
+  }
+
+  return true;
+}
+}; // namespace impl
+
+template <typename BitsetT, typename OffsetT>
+static void inclusive_scan(nvbench::state& state, nvbench::type_list<BitsetT, OffsetT>)
+{
+  using op_t         = impl::RabinKarpOp;
+  using input_t      = BitsetT;
+  using raw_it_t     = const input_t*;
+  using input_it_t   = cuda::transform_iterator<impl::ChunkToMat<input_t>, raw_it_t>;
+  using accum_t      = impl::MatT;
+  using output_ptr_t = impl::MatT*;
+  using output_it_t  = impl::write_at_specific_index_or_discard<OffsetT, output_ptr_t>;
+
+  using ZpT = impl::ZpT;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<input_t> input = generate(elements);
+  thrust::device_vector<accum_t> output(1, thrust::no_init);
+
+  // a large prime
+  ZpT p = static_cast<ZpT>(state.get_int64("Modulus"));
+
+  raw_it_t d_input      = thrust::raw_pointer_cast(input.data());
+  output_ptr_t d_output = thrust::raw_pointer_cast(output.data());
+
+  input_it_t inp_it(d_input, impl::ChunkToMat<input_t>(p));
+  output_it_t out_it(static_cast<OffsetT>(elements - 1), d_output);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<input_t>(elements, "Sequence Size");
+  state.add_global_memory_writes<accum_t>(1, "Hash Size");
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::InclusiveScan,
+      "InclusiveScan failed",
+      inp_it,
+      out_it, // iterator that only writes the last element of inclusive prefix scan sequence
+      op_t{p},
+      static_cast<OffsetT>(input.size()),
+      env);
+  });
+
+  // for validation uncomment these two lines
+  // assert(impl::validate(input, output, p, bench_stream));
+}
+
+#ifdef TUNE_T
+using type_list = nvbench::type_list<TUNE_T>;
+#else
+// we can split stream of bits into 8-bit, 16-bit, etc. chunks, effectively
+// serving as the number of bits processed by a thread
+using type_list = nvbench::type_list<cuda::std::uint8_t, cuda::std::uint16_t, cuda::std::uint32_t, cuda::std::uint64_t>;
+#endif
+
+NVBENCH_BENCH_TYPES(inclusive_scan, NVBENCH_TYPE_AXES(type_list, offset_types))
+  .set_name("rabin-karp-fingerprinting-monoid")
+  .set_type_axes_names({"BitsetT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("Modulus", {2725841});

cccl_upstream/cub/benchmarks/bench/scan/applications/P1/running-min-max.cu

@@ -0,0 +1,247 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/device_scan.cuh>
+
+#include <thrust/device_vector.h>
+#include <thrust/host_vector.h>
+
+#include <cuda/iterator>
+#include <cuda/std/cmath>
+#include <cuda/std/limits>
+#include <cuda/std/utility>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_DIRECT
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_WARP_TRANSPOSE
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+#endif // !TUNE_BASE
+
+#include "../../policy_selector.h"
+
+namespace impl
+{
+/* Given input sequence of values, compute sequence of
+ * pairs corresponding to running minimum and running maximum values.
+ */
+
+/*! @brief Structure to hold minimum and maximum */
+template <typename T>
+struct min_max_t
+{
+private:
+  T m_min{cuda::std::numeric_limits<T>::max()};
+  T m_max{cuda::std::numeric_limits<T>::min()};
+
+public:
+  min_max_t() = default;
+  __host__ __device__ min_max_t(T minimum, T maximum)
+      : m_min(minimum)
+      , m_max(maximum)
+  {}
+
+  T __host__ __device__ minimum() const
+  {
+    return m_min;
+  }
+  T __host__ __device__ maximum() const
+  {
+    return m_max;
+  }
+};
+
+/* Scan operator combining min-max pairs. It is commutative and associative */
+struct scan_op
+{
+  template <typename T>
+  min_max_t<T> __host__ __device__ operator()(min_max_t<T> v1, min_max_t<T> v2) const
+  {
+    auto min_r = cuda::minimum{}(v1.minimum(), v2.minimum());
+    auto max_r = cuda::maximum{}(v1.maximum(), v2.maximum());
+    return {min_r, max_r};
+  }
+};
+
+template <typename T>
+struct embed_op
+{
+  min_max_t<T> __host__ __device__ operator()(T v) const
+  {
+    return {v, v};
+  }
+};
+
+template <typename T>
+struct extract_min
+{
+  T __host__ __device__ operator()(min_max_t<T> pair) const
+  {
+    return pair.minimum();
+  }
+};
+
+template <typename T>
+struct extract_max
+{
+  T __host__ __device__ operator()(min_max_t<T> pair) const
+  {
+    return pair.maximum();
+  }
+};
+
+template <typename ValueT, typename PairT>
+void validate(const thrust::device_vector<ValueT>& input,
+              const thrust::device_vector<PairT>& output,
+              cudaStream_t stream)
+{
+  using value_t = ValueT;
+  auto elements = input.size();
+
+  thrust::device_vector<value_t> ref_mins(elements, thrust::no_init);
+  thrust::device_vector<value_t> ref_maxs(elements, thrust::no_init);
+
+  size_t tmp_size{};
+  auto d_input  = thrust::raw_pointer_cast(input.data());
+  auto d_output = thrust::raw_pointer_cast(output.data());
+
+  cub::DeviceScan::InclusiveScanInit(
+    nullptr,
+    tmp_size,
+    d_input,
+    ref_mins.begin(),
+    cuda::minimum<>{},
+    cuda::std::numeric_limits<value_t>::max(),
+    input.size(),
+    stream);
+
+  thrust::device_vector<nvbench::uint8_t> tmp1(tmp_size, thrust::no_init);
+  nvbench::uint8_t* d_tmp1 = thrust::raw_pointer_cast(tmp1.data());
+
+  cub::DeviceScan::InclusiveScanInit(
+    d_tmp1,
+    tmp_size,
+    d_input,
+    ref_mins.begin(),
+    cuda::minimum<>{},
+    cuda::std::numeric_limits<value_t>::max(),
+    input.size(),
+    stream);
+
+  cub::DeviceScan::InclusiveScanInit(
+    nullptr,
+    tmp_size,
+    d_input,
+    ref_maxs.begin(),
+    cuda::minimum<>{},
+    cuda::std::numeric_limits<value_t>::max(),
+    input.size(),
+    stream);
+
+  thrust::device_vector<nvbench::uint8_t> tmp2(tmp_size, thrust::no_init);
+  nvbench::uint8_t* d_tmp2 = thrust::raw_pointer_cast(tmp2.data());
+
+  cub::DeviceScan::InclusiveScanInit(
+    d_tmp2,
+    tmp_size,
+    d_input,
+    ref_maxs.begin(),
+    cuda::maximum<>{},
+    cuda::std::numeric_limits<value_t>::min(),
+    input.size(),
+    stream);
+
+  thrust::device_vector<value_t> computed_mins(elements, thrust::no_init);
+  thrust::device_vector<value_t> computed_maxs(elements, thrust::no_init);
+
+  impl::extract_min<value_t> extract_min_op{};
+  cub::DeviceTransform::Transform(d_output, computed_mins.begin(), input.size(), extract_min_op, stream);
+
+  impl::extract_max<value_t> extract_max_op{};
+  cub::DeviceTransform::Transform(d_output, computed_maxs.begin(), input.size(), extract_max_op, stream);
+
+  assert(computed_mins == ref_mins);
+  assert(computed_maxs == ref_maxs);
+}
+}; // namespace impl
+
+template <typename T, typename OffsetT>
+void benchmark_impl(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using value_t                  = T;
+  using pair_t                   = impl::min_max_t<value_t>;
+  using op_t                     = impl::scan_op;
+  using accum_t [[maybe_unused]] = pair_t;
+  using input_raw_t              = const value_t*;
+  using input_it_t               = cuda::transform_iterator<impl::embed_op<value_t>, input_raw_t>;
+  using output_it_t              = pair_t*;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<pair_t> output(elements);
+  thrust::device_vector<value_t> input = generate(elements);
+
+  input_raw_t d_input  = thrust::raw_pointer_cast(input.data());
+  output_it_t d_output = thrust::raw_pointer_cast(output.data());
+
+  input_it_t inp_it(d_input, impl::embed_op<value_t>{});
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<value_t>(elements, "Size");
+  state.add_global_memory_writes<pair_t>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::InclusiveScan,
+      "InclusiveScan failed",
+      inp_it,
+      d_output,
+      op_t{},
+      static_cast<OffsetT>(input.size()),
+      env);
+  });
+
+  // for verification use
+  // impl::validate(input, output, state.get_cuda_stream().get_stream());
+}
+
+#ifdef TUNE_T
+using bench_types = nvbench::type_list<TUNE_T>;
+#else
+using bench_types = nvbench::type_list<nvbench::uint32_t, nvbench::int64_t, nvbench::float32_t, nvbench::float64_t>;
+#endif
+
+NVBENCH_BENCH_TYPES(benchmark_impl, NVBENCH_TYPE_AXES(bench_types, offset_types))
+  .set_name("running-min-max")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/scan/applications/P1/scan-over-unitriangular-group.cu

@@ -0,0 +1,165 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/device_scan.cuh>
+
+#include <thrust/device_vector.h>
+#include <thrust/host_vector.h>
+
+#include <cuda/iterator>
+#include <cuda/std/tuple>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_DIRECT
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM  cub::BLOCK_LOAD_WARP_TRANSPOSE
+#    define TUNE_STORE_ALGORITHM cub::BLOCK_STORE_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+#endif // !TUNE_BASE
+
+#include "../../policy_selector.h"
+
+namespace impl
+{
+template <typename T>
+using triplet_t = cuda::std::tuple<T, T, T>;
+
+/* The triplet corresponds to strictly upper triangular elements of a unitriangular matrix
+   A = [[1, a1, a12], [0, 1, a2], [0, 0, 1]], mapped to triplet [a1, a2, a12].
+
+   The set of unitriangular matrix forms a group, with product induced by matrix multiplication,
+   and the identity element corresponding to zero triplet.
+*/
+struct unitriangular_dim3_op
+{
+  // Scan operation: associative and non-commutative
+  template <typename T>
+  triplet_t<T> __host__ __device__ operator()(triplet_t<T> a, triplet_t<T> b) const
+  {
+    auto [a1, a2, a12] = a;
+    auto [b1, b2, b12] = b;
+
+    return {a1 + b1, a2 + b1, a12 + b12 + a1 * b2};
+  }
+};
+
+// Utility operation to pack arguments into a triplet_t instance
+struct pack_op
+{
+  template <typename T>
+  triplet_t<T> __host__ __device__ operator()(T a1, T a2, T a12) const
+  {
+    return {a1, a2, a12};
+  } // namespace impl
+};
+
+template <typename TupleT, typename ScanOpT>
+bool validation(const thrust::device_vector<TupleT>& input,
+                const thrust::device_vector<TupleT>& output,
+                ScanOpT op,
+                cudaStream_t stream)
+{
+  cudaStreamSynchronize(stream);
+
+  using tuple_t = TupleT;
+  thrust::host_vector<tuple_t> h_input(input);
+  thrust::host_vector<tuple_t> h_output(output);
+
+  auto elements = input.size();
+  thrust::host_vector<tuple_t> h_reference(elements);
+
+  h_reference[0] = h_input[0];
+  for (std::size_t i = 1; i < elements; ++i)
+  {
+    h_reference[i] = op(h_reference[i - 1], h_input[i]);
+  }
+
+  return h_reference == h_output;
+}
+}; // namespace impl
+
+template <typename T, typename OffsetT>
+void benchmark_impl(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using value_t                  = T;
+  using tuple_t                  = impl::triplet_t<value_t>;
+  using op_t                     = impl::unitriangular_dim3_op;
+  using accum_t [[maybe_unused]] = tuple_t;
+
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  cudaStream_t bench_stream = state.get_cuda_stream().get_stream();
+
+  thrust::device_vector<tuple_t> output(elements);
+  thrust::device_vector<value_t> _input = generate(cuda::std::tuple_size_v<tuple_t> * elements);
+  thrust::device_vector<tuple_t> input(elements);
+
+  cub::DeviceTransform::Transform(
+    cuda::std::make_tuple(cuda::strided_iterator(_input.begin(), std::size_t{3}),
+                          cuda::strided_iterator(_input.begin() + 1, std::size_t{3}),
+                          cuda::strided_iterator(_input.begin() + 2, std::size_t{3})),
+    input.begin(),
+    input.size(),
+    impl::pack_op{},
+    bench_stream);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<tuple_t>(elements, "Size");
+  state.add_global_memory_writes<tuple_t>(elements);
+
+  auto d_input  = thrust::raw_pointer_cast(input.data());
+  auto d_output = thrust::raw_pointer_cast(output.data());
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::InclusiveScan,
+      "InclusiveScan failed",
+      d_input,
+      d_output,
+      op_t{},
+      static_cast<OffsetT>(input.size()),
+      env);
+  });
+
+  // for validation use (recommended for integral types and smallish input sizes)
+  // assert(impl::validation(input, output, op_t{}, bench_stream));
+}
+
+#ifdef TUNE_T
+using bench_types = nvbench::type_list<TUNE_T>;
+#else
+using bench_types = nvbench::type_list<nvbench::int32_t, nvbench::uint64_t, nvbench::float32_t, nvbench::float64_t>;
+#endif
+
+NVBENCH_BENCH_TYPES(benchmark_impl, NVBENCH_TYPE_AXES(bench_types, offset_types))
+  .set_name("unitriangular-monoid")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/scan/exclusive/base.cuh

@@ -0,0 +1,88 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#pragma once
+
+#include <cub/device/device_scan.cuh>
+
+#include <cuda/std/__functional/invoke.h>
+
+#include <nvbench_helper.cuh>
+
+#include "../policy_selector.h"
+
+template <typename T, typename OffsetT>
+static void basic(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+try
+{
+  using init_value_t             = T;
+  using accum_t [[maybe_unused]] = ::cuda::std::__accumulator_t<op_t, init_value_t, T>;
+  using offset_t                 = cub::detail::choose_offset_t<OffsetT>;
+#if USES_LOOKAHEAD()
+  static_assert(sizeof(offset_t) == sizeof(size_t)); // lookahead scan uses size_t internally
+#endif // USES_LOOKAHEAD()
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  if (sizeof(offset_t) == 4 && elements > std::numeric_limits<offset_t>::max())
+  {
+    state.skip("Skipping: input size exceeds 32-bit offset type capacity.");
+    return;
+  }
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements);
+
+  const T* d_input = thrust::raw_pointer_cast(input.data());
+  T* d_output      = thrust::raw_pointer_cast(output.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::ExclusiveScan,
+      "ExclusiveScan failed",
+      d_input,
+      d_output,
+      op_t{},
+      init_value_t{},
+      static_cast<offset_t>(input.size()),
+      env);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+// __half and __nv_bfloat16 are added for full (non-tuning) runs; CUB has fast paths for them (see #9587).
+#ifdef TUNE_T
+using value_types = nvbench::type_list<TUNE_T>;
+#else
+using value_types =
+  push_back_t<all_types
+#  if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
+              ,
+              __half
+#  endif
+#  if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
+              ,
+              __nv_bfloat16
+#  endif
+              >;
+#endif
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(value_types, scan_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/scan/exclusive/by_key.cu

@@ -0,0 +1,106 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_scan.cuh>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#if !TUNE_BASE
+struct bench_scan_by_key_policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ScanByKeyPolicy
+  {
+    return {cub::ScanByKeyAlgorithm::lookback,
+            {TUNE_THREADS,
+             TUNE_ITEMS,
+             TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE,
+             TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : cub::LOAD_CA,
+             TUNE_TRANSPOSE == 0 ? cub::BLOCK_STORE_DIRECT : cub::BLOCK_STORE_WARP_TRANSPOSE,
+             cub::BLOCK_SCAN_WARP_SCANS,
+             lookback_delay_policy}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename KeyT, typename ValueT, typename OffsetT>
+static void scan(nvbench::state& state, nvbench::type_list<KeyT, ValueT, OffsetT>)
+{
+  using init_value_t  = ValueT;
+  using op_t          = ::cuda::std::plus<>;
+  using equality_op_t = ::cuda::std::equal_to<>;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<ValueT> in_vals(elements);
+  thrust::device_vector<ValueT> out_vals(elements);
+  thrust::device_vector<KeyT> keys = generate.uniform.key_segments(elements, 0, 5200);
+
+  const KeyT* d_keys      = thrust::raw_pointer_cast(keys.data());
+  const ValueT* d_in_vals = thrust::raw_pointer_cast(in_vals.data());
+  ValueT* d_out_vals      = thrust::raw_pointer_cast(out_vals.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<ValueT>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_scan_by_key_policy_selector{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::ExclusiveScanByKey,
+      "ExclusiveScanByKey failed",
+      d_keys,
+      d_in_vals,
+      d_out_vals,
+      op_t{},
+      init_value_t{},
+      static_cast<OffsetT>(elements),
+      equality_op_t{},
+      env);
+  });
+}
+
+using some_offset_types = nvbench::type_list<nvbench::int32_t>;
+
+#ifdef TUNE_KeyT
+using key_types = nvbench::type_list<TUNE_KeyT>;
+#else // !defined(TUNE_KeyT)
+using key_types = all_types;
+#endif // TUNE_KeyT
+
+#ifdef TUNE_ValueT
+using value_types = nvbench::type_list<TUNE_ValueT>;
+#else // !defined(TUNE_ValueT)
+using value_types =
+  nvbench::type_list<int8_t,
+                     int16_t,
+                     int32_t,
+                     int64_t
+#  if _CCCL_HAS_INT128()
+                     ,
+                     int128_t
+#  endif
+                     >;
+#endif // TUNE_ValueT
+
+NVBENCH_BENCH_TYPES(scan, NVBENCH_TYPE_AXES(key_types, value_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/scan/exclusive/custom.cu

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+// This benchmark uses a custom operation, max_t, which is not known to CUB, so no operator specific optimizations and
+// tunings are performed.
+
+// Because CUB cannot detect this operator, we cannot add any tunings based on the results of this benchmark. Its main
+// use is to detect regressions.
+
+#include <nvbench_helper.cuh>
+
+#define USES_LOOKAHEAD() 0
+using op_t              = max_t;
+using scan_offset_types = offset_types;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/scan/exclusive/deterministic.cu

@@ -0,0 +1,57 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/device/device_scan.cuh>
+
+#include <cuda/__execution/determinism.h>
+#include <cuda/__execution/require.h>
+#include <cuda/std/__functional/invoke.h>
+
+#include <nvbench_helper.cuh>
+
+template <typename T, typename OffsetT>
+static void exclusive_scan(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+try
+{
+  using init_value_t = T;
+  using offset_t     = OffsetT;
+  using scan_op_t    = ::cuda::std::plus<T>;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements, thrust::no_init);
+
+  const T* d_input = thrust::raw_pointer_cast(input.data());
+  T* d_output      = thrust::raw_pointer_cast(output.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(alloc, launch, cuda::execution::require(cuda::execution::determinism::run_to_run));
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceScan::ExclusiveScan,
+      "ExclusiveScan failed",
+      d_input,
+      d_output,
+      scan_op_t{},
+      init_value_t{},
+      static_cast<offset_t>(elements),
+      env);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+using types   = nvbench::type_list<float, double>;
+using offsets = nvbench::type_list<int64_t>;
+
+NVBENCH_BENCH_TYPES(exclusive_scan, NVBENCH_TYPE_AXES(types, offsets))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/scan/exclusive/sum.cu

@@ -0,0 +1,22 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+// Tuning parameters found for signed integer types apply equally for unsigned integer types
+
+#include <nvbench_helper.cuh>
+
+// This benchmark tunes the old, non-lookahead scan implementation. Using it for benchmarking, will pick the lookahead
+// implementation on SM100+, but it's better to use the sum.lookahead.cu benchmark instead, which uses a single OffsetT.
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+
+#define USES_LOOKAHEAD() 0
+using op_t              = ::cuda::std::plus<>;
+using scan_offset_types = offset_types;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/scan/exclusive/sum.lookahead.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// This tunes the lookahead implementation of scan, which is only available on SM100+. It has entirely different tuning
+// parameters and is agnostic of the offset type. It is thus in a separate file, so we can continue to tune the old scan
+// implementation on older hardware architectures.
+
+#include <cuda/__cccl_config>
+
+#if _CCCL_PP_COUNT(__CUDA_ARCH_LIST__) != 1
+#  warning "This benchmark does not support being compiled for multiple architectures. Disabling it."
+#else // _CCCL_PP_COUNT(__CUDA_ARCH_LIST__) != 1
+
+#  if __CUDA_ARCH_LIST__ < 1000
+// We don't care if clang-tidy can't parse this
+#    ifndef _CCCL_CLANG_TIDY_INVOKED
+#      warning "Lookahead scan requires at least sm_100. Disabling it."
+#    endif // !defined _CCCL_CLANG_TIDY_INVOKED
+#  else // __CUDA_ARCH_LIST__ < 1000
+
+#    if __cccl_ptx_isa < 860
+#      warning "Lookahead scan requires at least PTX ISA 8.6. Disabling it."
+#    else // if __cccl_ptx_isa < 860
+
+#      include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_NUM_REDUCE_SCAN_WARPS wrps 1:8:1
+// %RANGE% TUNE_NUM_LOOKBACK_ITEMS lbi 1:8:1
+
+// TODO(bgruber): find a good range and step width, items per thread should be coprime with 32 to avoid SMEM conflicts.
+// Should we specify nominal items per thread instead?
+// %RANGE% TUNE_ITEMS_PLUS_ONE ipt 8:256:8
+
+// %RANGE% TUNE_LOOKBACK_STAGES lbs -2:2:1
+// %RANGE% TUNE_BLOCK_IDX_STAGES bis -2:2:1
+
+#      define USES_LOOKAHEAD() 1
+using op_t              = ::cuda::std::plus<>;
+using scan_offset_types = nvbench::type_list<int64_t>;
+#      include "base.cuh"
+
+#    endif // __cccl_ptx_isa < 860
+#  endif // __CUDA_ARCH_LIST__ < 1000
+#endif // _CCCL_PP_COUNT(__CUDA_ARCH_LIST__) != 1

cccl_upstream/cub/benchmarks/bench/scan/policy_selector.h

@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/device/device_scan.cuh>
+
+#ifndef USES_LOOKAHEAD
+#  define USES_LOOKAHEAD() 0
+#endif
+
+#if !TUNE_BASE
+#  if !USES_LOOKAHEAD()
+#    include <look_back_helper.cuh>
+#  endif // !USES_LOOKAHEAD()
+
+template <typename AccumT>
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ScanPolicy
+  {
+#  if USES_LOOKAHEAD()
+    return {cub::ScanAlgorithm::lookahead,
+            cub::ScanLookbackPolicy{},
+            cub::ScanLookaheadPolicy{
+              TUNE_NUM_REDUCE_SCAN_WARPS,
+              TUNE_ITEMS_PLUS_ONE - 1,
+              TUNE_NUM_LOOKBACK_ITEMS,
+              TUNE_LOOKBACK_STAGES,
+              TUNE_BLOCK_IDX_STAGES}};
+#  else
+    return cub::detail::scan::make_mem_scaled_lookback_scan_policy(
+      TUNE_THREADS,
+      TUNE_ITEMS,
+      int{sizeof(AccumT)},
+      TUNE_LOAD_ALGORITHM,
+      TUNE_LOAD_MODIFIER,
+      TUNE_STORE_ALGORITHM,
+      cub::BLOCK_SCAN_WARP_SCANS,
+      lookback_delay_policy);
+#  endif
+  }
+};
+#endif // !TUNE_BASE

cccl_upstream/cub/benchmarks/bench/segmented_radix_sort/keys.cu

@@ -0,0 +1,109 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+// This benchmark is only used for regression testing and not tuning
+
+#include <cub/device/device_segmented_radix_sort.cuh>
+
+#include <nvbench_helper.cuh>
+
+template <class T, typename OffsetT>
+void seg_radix_sort(nvbench::state& state,
+                    nvbench::type_list<T, OffsetT>,
+                    const thrust::device_vector<OffsetT>& offsets,
+                    bit_entropy entropy)
+{
+  using offset_t          = OffsetT;
+  using begin_offset_it_t = const offset_t*;
+  using end_offset_it_t   = const offset_t*;
+  using key_t             = T;
+
+  constexpr int begin_bit = 0;
+  constexpr int end_bit   = sizeof(key_t) * 8;
+
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const auto segments = offsets.size() - 1;
+
+  thrust::device_vector<key_t> buffer_1 = generate(elements, entropy);
+  thrust::device_vector<key_t> buffer_2(elements, thrust::no_init);
+
+  const key_t* d_keys_1 = thrust::raw_pointer_cast(buffer_1.data());
+  key_t* d_keys_2       = thrust::raw_pointer_cast(buffer_2.data());
+
+  begin_offset_it_t d_begin_offsets = thrust::raw_pointer_cast(offsets.data());
+  end_offset_it_t d_end_offsets     = d_begin_offsets + 1;
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<key_t>(elements);
+  state.add_global_memory_writes<key_t>(elements);
+  state.add_global_memory_reads<offset_t>(segments + 1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               const auto env = cub_bench_env(alloc, launch);
+               _CCCL_TRY_CUDA_API(
+                 cub::DeviceSegmentedRadixSort::SortKeys,
+                 "SortKeys failed",
+                 d_keys_1,
+                 d_keys_2,
+                 elements,
+                 segments,
+                 d_begin_offsets,
+                 d_end_offsets,
+                 begin_bit,
+                 end_bit,
+                 env);
+             });
+}
+
+#ifdef TUNE_OffsetT
+using some_offset_types = nvbench::type_list<TUNE_OffsetT>;
+#else
+using some_offset_types = nvbench::type_list<int32_t, int64_t>;
+#endif
+
+template <class T, typename OffsetT>
+void power_law(nvbench::state& state, nvbench::type_list<T, OffsetT> ts)
+{
+  const auto elements                    = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const auto segments                    = static_cast<std::size_t>(state.get_int64("Segments{io}"));
+  const bit_entropy entropy              = str_to_entropy(state.get_string("Entropy"));
+  thrust::device_vector<OffsetT> offsets = generate.power_law.segment_offsets(elements, segments);
+
+  seg_radix_sort(state, ts, offsets, entropy);
+}
+
+NVBENCH_BENCH_TYPES(power_law, NVBENCH_TYPE_AXES(fundamental_types, some_offset_types))
+  .set_name("power")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(22, 30, 4))
+  .add_int64_power_of_two_axis("Segments{io}", nvbench::range(12, 20, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"});
+
+template <class T, typename OffsetT>
+void uniform(nvbench::state& state, nvbench::type_list<T, OffsetT> ts)
+{
+  const auto elements         = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const auto max_segment_size = static_cast<std::size_t>(state.get_int64("MaxSegmentSize"));
+
+  const auto max_segment_size_log = static_cast<OffsetT>(std::log2(max_segment_size));
+  const auto min_segment_size     = 1 << (max_segment_size_log - 1);
+
+  thrust::device_vector<OffsetT> offsets =
+    generate.uniform.segment_offsets(elements, min_segment_size, max_segment_size);
+
+  seg_radix_sort(state, ts, offsets, bit_entropy::_1_000);
+}
+
+NVBENCH_BENCH_TYPES(uniform, NVBENCH_TYPE_AXES(fundamental_types, some_offset_types))
+  .set_name("small")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(22, 30, 4))
+  .add_int64_power_of_two_axis("MaxSegmentSize", nvbench::range(1, 8, 1));
+
+NVBENCH_BENCH_TYPES(uniform, NVBENCH_TYPE_AXES(fundamental_types, some_offset_types))
+  .set_name("large")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(22, 30, 4))
+  .add_int64_power_of_two_axis("MaxSegmentSize", nvbench::range(10, 18, 2));

cccl_upstream/cub/benchmarks/bench/segmented_reduce/argmin.cu

@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_VEC_LOAD_POW2 ipv 1:2:1
+// %RANGE% TUNE_S_THREADS_PER_WARP stpw 1:32:1
+// %RANGE% TUNE_M_THREADS_PER_WARP mtpw 1:32:1
+// %RANGE% TUNE_L_NOMINAL_4B_THREADS_PER_BLOCK ltpb 128:1024:32
+// %RANGE% TUNE_S_NOMINAL_4B_ITEMS_PER_THREAD sipt 1:32:1
+// %RANGE% TUNE_M_NOMINAL_4B_ITEMS_PER_THREAD mipt 1:32:1
+// %RANGE% TUNE_L_NOMINAL_4B_ITEMS_PER_THREAD lipt 7:24:1
+
+using value_types = integral_types;
+using op_t        = cub::detail::arg_min;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/segmented_reduce/base.cuh

@@ -0,0 +1,122 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#pragma once
+
+#include <cub/device/device_segmented_reduce.cuh>
+
+#include <cuda/std/type_traits>
+
+#ifndef TUNE_BASE
+#  define TUNE_ITEMS_PER_VEC_LOAD (1 << TUNE_ITEMS_PER_VEC_LOAD_POW2)
+#endif
+
+#if !TUNE_BASE
+template <typename AccumT>
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const
+    -> ::cub::SegmentedReducePolicy
+  {
+    constexpr int accum_size = int{sizeof(AccumT)};
+
+    const auto [l_items, l_threads] =
+      cub::detail::scale_mem_bound(TUNE_L_NOMINAL_4B_THREADS_PER_BLOCK, TUNE_L_NOMINAL_4B_ITEMS_PER_THREAD, accum_size);
+    const auto s_items =
+      cub::detail::scale_mem_bound(TUNE_L_NOMINAL_4B_THREADS_PER_BLOCK, TUNE_S_NOMINAL_4B_ITEMS_PER_THREAD, accum_size)
+        .items_per_thread;
+    const auto m_items =
+      cub::detail::scale_mem_bound(TUNE_L_NOMINAL_4B_THREADS_PER_BLOCK, TUNE_M_NOMINAL_4B_ITEMS_PER_THREAD, accum_size)
+        .items_per_thread;
+
+    const auto rp = cub::ReducePassPolicy{
+      l_threads, l_items, TUNE_ITEMS_PER_VEC_LOAD, cub::BLOCK_REDUCE_WARP_REDUCTIONS, cub::LOAD_LDG};
+    return {rp,
+            cub::SegmentedReduceWarpReducePolicy{
+              rp.threads_per_block, TUNE_M_THREADS_PER_WARP, m_items, rp.vec_size, rp.load_modifier},
+            cub::SegmentedReduceWarpReducePolicy{
+              rp.threads_per_block, TUNE_S_THREADS_PER_WARP, s_items, rp.vec_size, rp.load_modifier}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T>
+void fixed_size_segmented_reduce(nvbench::state& state, nvbench::type_list<T>)
+{
+  static constexpr bool is_argmin = std::is_same_v<op_t, cub::detail::arg_min>;
+
+  using output_t     = cuda::std::conditional_t<is_argmin, cuda::std::pair<int, T>, T>;
+  using accum_t      = output_t;
+  using init_value_t = cuda::std::conditional_t<is_argmin, cub::detail::reduce::empty_problem_init_t<accum_t>, T>;
+
+  // Retrieve axis parameters
+  const size_t num_elements = static_cast<size_t>(state.get_int64("Elements{io}"));
+  const size_t segment_size = static_cast<size_t>(state.get_int64("SegmentSize"));
+  const size_t num_segments = std::max<std::size_t>(1, (num_elements / segment_size));
+  const size_t elements     = num_segments * segment_size;
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<output_t> out(num_segments);
+
+  const T* d_in   = thrust::raw_pointer_cast(in.data());
+  output_t* d_out = thrust::raw_pointer_cast(out.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<output_t>(num_segments);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<accum_t>{})
+#endif
+    );
+    if constexpr (is_argmin)
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSegmentedReduce::ArgMin,
+        "Segmented ArgMin failed",
+        d_in,
+        d_out,
+        static_cast<::cuda::std::int64_t>(num_segments),
+        static_cast<int>(segment_size),
+        env);
+    }
+    else
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSegmentedReduce::Reduce,
+        "Segmented reduce failed",
+        d_in,
+        d_out,
+        static_cast<::cuda::std::int64_t>(num_segments),
+        static_cast<int>(segment_size),
+        op_t{},
+        init_value_t{},
+        env);
+    }
+  });
+}
+
+NVBENCH_BENCH_TYPES(fixed_size_segmented_reduce, NVBENCH_TYPE_AXES(value_types))
+  .set_name("small")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("SegmentSize", nvbench::range(0, 4, 1));
+
+NVBENCH_BENCH_TYPES(fixed_size_segmented_reduce, NVBENCH_TYPE_AXES(value_types))
+  .set_name("medium")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("SegmentSize", nvbench::range(5, 8, 1));
+
+NVBENCH_BENCH_TYPES(fixed_size_segmented_reduce, NVBENCH_TYPE_AXES(value_types))
+  .set_name("large")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("SegmentSize", nvbench::range(9, 16, 1));

cccl_upstream/cub/benchmarks/bench/segmented_reduce/custom.cu

@@ -0,0 +1,8 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <nvbench_helper.cuh>
+
+using value_types = all_types;
+using op_t        = max_t;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/segmented_reduce/sum.cu

@@ -0,0 +1,16 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_VEC_LOAD_POW2 ipv 1:2:1
+// %RANGE% TUNE_S_THREADS_PER_WARP stpw 1:32:1
+// %RANGE% TUNE_M_THREADS_PER_WARP mtpw 1:32:1
+// %RANGE% TUNE_L_NOMINAL_4B_THREADS_PER_BLOCK ltpb 128:1024:32
+// %RANGE% TUNE_S_NOMINAL_4B_ITEMS_PER_THREAD sipt 1:32:1
+// %RANGE% TUNE_M_NOMINAL_4B_ITEMS_PER_THREAD mipt 1:32:1
+// %RANGE% TUNE_L_NOMINAL_4B_ITEMS_PER_THREAD lipt 7:24:1
+
+using value_types = all_types;
+using op_t        = ::cuda::std::plus<>;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/segmented_reduce/variable_argmax.cu

@@ -0,0 +1,8 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <nvbench_helper.cuh>
+
+using op_t = cub::detail::arg_max;
+
+#include "variable_base.cuh"

cccl_upstream/cub/benchmarks/bench/segmented_reduce/variable_base.cuh

@@ -0,0 +1,161 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#pragma once
+
+#include <cub/device/dispatch/dispatch_segmented_reduce.cuh>
+
+#include <cuda/std/iterator>
+#include <cuda/std/type_traits>
+
+#include <nvbench_helper.cuh>
+
+#if TUNE_T
+using value_types = nvbench::type_list<TUNE_T>;
+#else
+using value_types = nvbench::type_list<int32_t, int64_t, float, double>;
+#endif
+
+#ifdef TUNE_OffsetT
+using some_offset_types = nvbench::type_list<TUNE_OffsetT>;
+#else
+using some_offset_types = nvbench::type_list<int32_t>;
+#endif
+
+template <typename T, typename OffsetT>
+void variable_segmented_reduce(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  static constexpr bool is_argmin = std::is_same_v<op_t, cub::detail::arg_min>;
+  static constexpr bool is_argmax = std::is_same_v<op_t, cub::detail::arg_max>;
+
+  using raw_input_it_t = const T*;
+  using output_t       = cuda::std::conditional_t<(is_argmin || is_argmax), cuda::std::pair<int, T>, T>;
+  using output_it_t    = output_t*;
+  using accum_t        = output_t;
+  using init_value_t =
+    cuda::std::conditional_t<(is_argmin || is_argmax), cub::detail::reduce::empty_problem_init_t<accum_t>, T>;
+  using offset_t          = OffsetT;
+  using begin_offset_it_t = const offset_t*;
+  using end_offset_it_t   = const offset_t*;
+
+  // Retrieve axis parameters
+  const auto elements                = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const auto max_segment_size        = static_cast<std::size_t>(state.get_int64("MaxSegmentSize"));
+  const auto guaranteed_max_seg_size = static_cast<std::size_t>(state.get_int64("GuaranteedMaxSegSize"));
+
+  // skip if max_segment_size > guaranteed_max_seg_size
+  if (guaranteed_max_seg_size != 0 && max_segment_size > guaranteed_max_seg_size)
+  {
+    state.skip("max_segment_size > guaranteed_max_seg_size");
+    return;
+  }
+
+  const auto min_segment_size     = 1;
+  const auto max_segment_size_log = static_cast<offset_t>(std::log2(max_segment_size));
+
+  // Generate segment offsets
+  thrust::device_vector<offset_t> segment_offsets =
+    generate.uniform.segment_offsets(elements, min_segment_size, max_segment_size);
+  const auto num_segments = segment_offsets.size() - 1;
+
+  // Generate input data
+  thrust::device_vector<T> in = generate(elements);
+
+  thrust::device_vector<output_t> out(num_segments, thrust::default_init);
+
+  raw_input_it_t d_raw_in           = thrust::raw_pointer_cast(in.data());
+  output_it_t d_out                 = thrust::raw_pointer_cast(out.data());
+  begin_offset_it_t d_begin_offsets = thrust::raw_pointer_cast(segment_offsets.data());
+  end_offset_it_t d_end_offsets     = d_begin_offsets + 1;
+
+  // Create wrapped iterator for argmin/argmax operations
+  [[maybe_unused]] auto d_indexed_in = cuda::make_transform_iterator(
+    cuda::counting_iterator<::cuda::std::int64_t>(0),
+    cub::detail::segmented_reduce::generate_idx_value<raw_input_it_t, T>(d_raw_in, 1));
+  using arg_index_input_iterator_t = decltype(d_indexed_in);
+
+  auto d_in = [&] {
+    if constexpr (is_argmin || is_argmax)
+    {
+      return d_indexed_in;
+    }
+    else
+    {
+      return d_raw_in;
+    }
+  }();
+
+  // Enable throughput calculations
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<output_t>(num_segments);
+  state.add_global_memory_reads<offset_t>(num_segments + 1);
+
+  // Allocate temporary storage
+  std::size_t temp_size{};
+  using override_offset_t = cuda::std::conditional_t<(is_argmin || is_argmax), int, cub::detail::use_default>;
+
+  // TODO(bgruber): rewrite this to use the public CUB API directly. But in order to do this, we need to expose the
+  // guaranteed_max_seg_size at the public API
+  cub::detail::segmented_reduce::dispatch<accum_t, override_offset_t>(
+    nullptr,
+    temp_size,
+    d_in,
+    d_out,
+    static_cast<::cuda::std::int64_t>(num_segments),
+    d_begin_offsets,
+    d_end_offsets,
+    op_t{},
+    init_value_t{},
+    guaranteed_max_seg_size,
+    nullptr /* stream */);
+
+  thrust::device_vector<nvbench::uint8_t> temp(temp_size, thrust::no_init);
+  auto* temp_storage = thrust::raw_pointer_cast(temp.data());
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    cub::detail::segmented_reduce::dispatch<accum_t, override_offset_t>(
+      temp_storage,
+      temp_size,
+      d_in,
+      d_out,
+      static_cast<::cuda::std::int64_t>(num_segments),
+      d_begin_offsets,
+      d_end_offsets,
+      op_t{},
+      init_value_t{},
+      guaranteed_max_seg_size,
+      launch.get_stream());
+  });
+}
+
+NVBENCH_BENCH_TYPES(variable_segmented_reduce, NVBENCH_TYPE_AXES(value_types, some_offset_types))
+  .set_name("variable_default")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegmentSize", nvbench::range(1, 16, 1))
+  .add_int64_axis("GuaranteedMaxSegSize", {0});
+
+// Small segments: 1-16 items per segment
+NVBENCH_BENCH_TYPES(variable_segmented_reduce, NVBENCH_TYPE_AXES(value_types, some_offset_types))
+  .set_name("variable_small_dynamic")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegmentSize", nvbench::range(1, 4, 1))
+  .add_int64_power_of_two_axis("GuaranteedMaxSegSize", nvbench::range(1, 4, 1));
+
+// Medium segments: 32-256 items per segment
+NVBENCH_BENCH_TYPES(variable_segmented_reduce, NVBENCH_TYPE_AXES(value_types, some_offset_types))
+  .set_name("variable_medium_dynamic")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegmentSize", nvbench::range(5, 8, 1))
+  .add_int64_power_of_two_axis("GuaranteedMaxSegSize", nvbench::range(5, 8, 1));
+
+// Large segments: 512+ items per segment
+NVBENCH_BENCH_TYPES(variable_segmented_reduce, NVBENCH_TYPE_AXES(value_types, some_offset_types))
+  .set_name("variable_large_dynamic")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegmentSize", nvbench::range(9, 16, 1))
+  .add_int64_power_of_two_axis("GuaranteedMaxSegSize", nvbench::range(9, 16, 1));

cccl_upstream/cub/benchmarks/bench/segmented_reduce/variable_sum.cu

@@ -0,0 +1,8 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <nvbench_helper.cuh>
+
+using op_t = ::cuda::std::plus<>;
+
+#include "variable_base.cuh"

cccl_upstream/cub/benchmarks/bench/segmented_scan/base.cuh

@@ -0,0 +1,153 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#pragma once
+
+#include <cub/device/device_segmented_scan.cuh>
+
+#include <thrust/tabulate.h>
+
+#include <cuda/std/__functional/invoke.h>
+#include <cuda/std/type_traits>
+
+#include <nvbench_helper.cuh>
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_BLOCK_LOAD_ALGORITHM  cub::BLOCK_LOAD_DIRECT
+#    define TUNE_BLOCK_STORE_ALGORITHM cub::BLOCK_STORE_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_BLOCK_LOAD_ALGORITHM  cub::BLOCK_LOAD_WARP_TRANSPOSE
+#    define TUNE_BLOCK_STORE_ALGORITHM cub::BLOCK_STORE_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+
+template <int ThreadsPerBlock, int ItemsPerThread, int MaxSegmentsPerBlock>
+struct policy_selector_t
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::SegmentedScanPolicy
+  {
+    return cub::SegmentedScanPolicy{cub::SegmentedScanBlockPolicy{
+      ThreadsPerBlock,
+      ItemsPerThread,
+      TUNE_BLOCK_LOAD_ALGORITHM,
+      TUNE_LOAD_MODIFIER,
+      TUNE_BLOCK_STORE_ALGORITHM,
+      cub::BLOCK_SCAN_WARP_SCANS,
+      MaxSegmentsPerBlock}};
+  }
+};
+#endif // TUNE_BASE
+
+template <typename OffsetT>
+struct to_offsets_functor
+{
+  OffsetT elements;
+  OffsetT segment_size;
+  OffsetT wobble;
+
+  __host__ __device__ __forceinline__ OffsetT operator()(size_t i) const
+  {
+    const auto fixed_size_value = static_cast<OffsetT>(i) * segment_size;
+    const auto correction       = ((i & 1) ? wobble : OffsetT{0});
+
+    return cuda::std::min(elements, fixed_size_value + correction);
+  }
+};
+
+template <size_t Wobble = 0, typename T, typename OffsetT>
+static void bench_impl(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+#if !TUNE_BASE
+  using policy_t = policy_selector_t<TUNE_THREADS, TUNE_ITEMS, TUNE_MAX_SEGMENTS_PER_BLOCK>;
+#endif
+
+  const auto elements     = static_cast<OffsetT>(state.get_int64("Elements{io}"));
+  const auto segment_size = static_cast<OffsetT>(state.get_int64("SegmentSize{io}"));
+  const auto num_segments = cuda::ceil_div(elements, segment_size);
+  auto& summary           = state.add_summary("user/derived/segment_count");
+  summary.set_string("name", "#Segments");
+  summary.set_int64("value", num_segments);
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements, thrust::default_init);
+
+  thrust::device_vector<OffsetT> offsets(num_segments + 1, thrust::no_init);
+  thrust::tabulate(offsets.begin(), offsets.end(), to_offsets_functor<OffsetT>{elements, segment_size, Wobble});
+
+  const T* d_input         = thrust::raw_pointer_cast(input.data());
+  T* d_output              = thrust::raw_pointer_cast(output.data());
+  const OffsetT* d_offsets = thrust::raw_pointer_cast(offsets.data());
+
+  state.add_element_count(elements, "Elements");
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_reads<OffsetT>(num_segments + 1);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_t{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceSegmentedScan::ExclusiveSegmentedScan,
+      "ExclusiveSegmentedScan failed",
+      d_input,
+      d_output,
+      d_offsets,
+      d_offsets + 1,
+      d_offsets,
+      num_segments,
+      op_t{},
+      T{},
+      env);
+  });
+}
+
+template <typename T, typename OffsetT>
+static void fixed_segment_size_bench(nvbench::state& state, nvbench::type_list<T, OffsetT> tl)
+{
+  return bench_impl<0, T, OffsetT>(state, tl);
+}
+
+template <typename T, typename OffsetT>
+static void varying_segment_size_bench(nvbench::state& state, nvbench::type_list<T, OffsetT> tl)
+{
+  return bench_impl<1, T, OffsetT>(state, tl);
+}
+
+#if (_CCCL_CUDA_COMPILER(NVCC, >=, 12, 1))
+using benched_value_types = all_types;
+#else
+// WAR for excessive time CTK 12.0 CICC takes to compile these benchmarks for int128_t
+#  ifdef TUNE_T
+static_assert(!cuda::std::is_integral_v<TUNE_T> || sizeof(TUNE_T) < 16);
+using benched_value_types = nvbench::type_list<TUNE_T>;
+#  else
+using benched_value_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double, complex32>;
+#  endif
+#endif
+
+NVBENCH_BENCH_TYPES(fixed_segment_size_bench, NVBENCH_TYPE_AXES(benched_value_types, offset_types))
+  .set_name("fixed_size_segments")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(18, 26, 4))
+  .add_int64_axis("SegmentSize{io}", {51, 123, 233, 513, 1337, 4417});
+
+NVBENCH_BENCH_TYPES(varying_segment_size_bench, NVBENCH_TYPE_AXES(benched_value_types, offset_types))
+  .set_name("varying_size_segments")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(18, 26, 4))
+  .add_int64_axis("SegmentSize{io}", {51, 123, 233, 513, 1337, 4417});
+// .add_int64_axis("SegmentsPerWorker{io}", {1}) // public API doesn' expose them (yet)
+// .add_string_axis("Worker{io}", {"block"});

cccl_upstream/cub/benchmarks/bench/segmented_scan/custom.cu

@@ -0,0 +1,13 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// This benchmark uses a custom operation, max_t, which is not known to CUB, so no operator specific optimizations and
+// tunings are performed.
+
+// Because CUB cannot detect this operator, we cannot add any tunings based on the results of this benchmark. Its main
+// use is to detect regressions.
+
+#include <nvbench_helper.cuh>
+
+using op_t = max_t;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/segmented_scan/sum.cu

@@ -0,0 +1,15 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Tuning parameters found for signed integer types apply equally for unsigned integer types
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_MAX_SEGMENTS_PER_BLOCK spb 1:512:511
+
+using op_t = ::cuda::std::plus<>;
+#include "base.cuh"

cccl_upstream/cub/benchmarks/bench/segmented_sort/keys.cu

@@ -0,0 +1,169 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_segmented_sort.cuh>
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_L_ITEMS ipt 7:24:1
+// %RANGE% TUNE_M_ITEMS ipmw 1:17:1
+// %RANGE% TUNE_S_ITEMS ipsw 1:17:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_SW_THREADS_POW2 tpsw 1:4:1
+// %RANGE% TUNE_MW_THREADS_POW2 tpmw 1:5:1
+// %RANGE% TUNE_RADIX_BITS bits 4:8:1
+// %RANGE% TUNE_PARTITIONING_THRESHOLD pt 100:800:50
+// %RANGE% TUNE_RANK_ALGORITHM ra 0:4:1
+// %RANGE% TUNE_LOAD ld 0:2:1
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_S_LOAD sld 0:2:1
+// %RANGE% TUNE_S_TRANSPOSE strp 0:1:1
+// %RANGE% TUNE_M_LOAD mld 0:2:1
+// %RANGE% TUNE_M_TRANSPOSE mtrp 0:1:1
+
+#if !TUNE_BASE
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const
+  {
+    constexpr int tune_sw_threads     = 1 << TUNE_SW_THREADS_POW2;
+    constexpr int tune_mw_threads     = 1 << TUNE_MW_THREADS_POW2;
+    constexpr int small_segment_size  = TUNE_S_ITEMS * tune_sw_threads;
+    constexpr int medium_segment_size = TUNE_M_ITEMS * tune_mw_threads;
+    constexpr int large_segment_size  = TUNE_L_ITEMS * TUNE_THREADS;
+
+    static_assert((large_segment_size > small_segment_size) && (large_segment_size > medium_segment_size),
+                  "Large segment size must be larger than small and medium segment sizes");
+    static_assert(medium_segment_size > small_segment_size, "Medium segment size must be larger than small one");
+
+    return cub::SegmentedSortPolicy{
+      cub::SegmentedSortRadixSortPolicy{
+        TUNE_THREADS,
+        TUNE_L_ITEMS,
+        (TUNE_TRANSPOSE == 0) ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE,
+        (TUNE_LOAD == 0)   ? cub::LOAD_DEFAULT
+        : (TUNE_LOAD == 1) ? cub::LOAD_LDG
+                           : cub::LOAD_CA,
+        static_cast<cub::RadixRankAlgorithm>(TUNE_RANK_ALGORITHM),
+        cub::BLOCK_SCAN_WARP_SCANS,
+        TUNE_RADIX_BITS,
+      },
+      cub::SegmentedSortSubWarpMergeSortPolicy{
+        TUNE_THREADS,
+        tune_mw_threads,
+        TUNE_M_ITEMS,
+        (TUNE_M_TRANSPOSE == 0) ? cub::WarpLoadAlgorithm::WARP_LOAD_DIRECT : cub::WarpLoadAlgorithm::WARP_LOAD_TRANSPOSE,
+        (TUNE_M_LOAD == 0)   ? cub::LOAD_DEFAULT
+        : (TUNE_M_LOAD == 1) ? cub::LOAD_LDG
+                             : cub::LOAD_CA,
+        cub::WARP_STORE_DIRECT,
+      },
+      cub::SegmentedSortSubWarpMergeSortPolicy{
+        TUNE_THREADS,
+        tune_sw_threads,
+        TUNE_S_ITEMS,
+        (TUNE_S_TRANSPOSE == 0) ? cub::WarpLoadAlgorithm::WARP_LOAD_DIRECT : cub::WarpLoadAlgorithm::WARP_LOAD_TRANSPOSE,
+        (TUNE_S_LOAD == 0)   ? cub::LOAD_DEFAULT
+        : (TUNE_S_LOAD == 1) ? cub::LOAD_LDG
+                             : cub::LOAD_CA,
+        cub::WARP_STORE_DIRECT,
+      },
+      TUNE_PARTITIONING_THRESHOLD,
+    };
+  }
+};
+#endif // !TUNE_BASE
+
+template <class T, typename OffsetT>
+void seg_sort(nvbench::state& state,
+              nvbench::type_list<T, OffsetT>,
+              const thrust::device_vector<OffsetT>& offsets,
+              bit_entropy entropy)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const auto segments = offsets.size() - 1;
+
+  thrust::device_vector<T> buffer_1 = generate(elements, entropy);
+  thrust::device_vector<T> buffer_2(elements, thrust::no_init);
+
+  T* d_buffer_1 = thrust::raw_pointer_cast(buffer_1.data());
+  T* d_buffer_2 = thrust::raw_pointer_cast(buffer_2.data());
+
+  const OffsetT* d_begin_offsets = thrust::raw_pointer_cast(offsets.data());
+  const OffsetT* d_end_offsets   = d_begin_offsets + 1;
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+  state.add_global_memory_reads<OffsetT>(segments + 1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               auto env = cub_bench_env(
+                 alloc,
+                 launch
+#if !TUNE_BASE
+                 ,
+                 cuda::execution::tune(policy_selector{})
+#endif // !TUNE_BASE
+               );
+               _CCCL_TRY_CUDA_API(
+                 cub::DeviceSegmentedSort::SortKeys,
+                 "SortKeys failed",
+                 d_buffer_1,
+                 d_buffer_2,
+                 static_cast<cuda::std::int64_t>(elements),
+                 static_cast<cuda::std::int64_t>(segments),
+                 d_begin_offsets,
+                 d_end_offsets,
+                 env);
+             });
+}
+
+using some_offset_types = nvbench::type_list<int32_t>;
+
+template <class T, typename OffsetT>
+void power_law(nvbench::state& state, nvbench::type_list<T, OffsetT> ts)
+{
+  const auto elements                    = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const auto segments                    = static_cast<std::size_t>(state.get_int64("Segments{io}"));
+  const bit_entropy entropy              = str_to_entropy(state.get_string("Entropy"));
+  thrust::device_vector<OffsetT> offsets = generate.power_law.segment_offsets(elements, segments);
+
+  seg_sort(state, ts, offsets, entropy);
+}
+
+NVBENCH_BENCH_TYPES(power_law, NVBENCH_TYPE_AXES(fundamental_types, some_offset_types))
+  .set_name("power")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(22, 30, 4))
+  .add_int64_power_of_two_axis("Segments{io}", nvbench::range(12, 20, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"});
+
+template <class T, typename OffsetT>
+void uniform(nvbench::state& state, nvbench::type_list<T, OffsetT> ts)
+{
+  const auto elements         = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  const auto max_segment_size = static_cast<std::size_t>(state.get_int64("MaxSegmentSize"));
+
+  const auto max_segment_size_log = static_cast<OffsetT>(std::log2(max_segment_size));
+  const auto min_segment_size     = 1 << (max_segment_size_log - 1);
+
+  thrust::device_vector<OffsetT> offsets =
+    generate.uniform.segment_offsets(elements, min_segment_size, max_segment_size);
+
+  seg_sort(state, ts, offsets, bit_entropy::_1_000);
+}
+
+NVBENCH_BENCH_TYPES(uniform, NVBENCH_TYPE_AXES(fundamental_types, some_offset_types))
+  .set_name("small")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(22, 30, 4))
+  .add_int64_power_of_two_axis("MaxSegmentSize", nvbench::range(1, 8, 1));
+
+NVBENCH_BENCH_TYPES(uniform, NVBENCH_TYPE_AXES(fundamental_types, some_offset_types))
+  .set_name("large")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(22, 30, 4))
+  .add_int64_power_of_two_axis("MaxSegmentSize", nvbench::range(10, 18, 2));

cccl_upstream/cub/benchmarks/bench/segmented_topk/fixed/keys.cu

@@ -0,0 +1,122 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/dispatch/dispatch_batched_topk.cuh>
+
+#include <cuda/argument>
+#include <cuda/iterator>
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 1:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_BLOCK_LOAD_ALGORITHM ld 0:2:1
+
+#if !TUNE_BASE
+struct tuned_policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const
+    -> cub::detail::batched_topk::batched_topk_policy
+  {
+    // Single-entry policy chain driven by the tuning knobs.
+    constexpr auto store_alg = cub::BLOCK_STORE_WARP_TRANSPOSE;
+#  if TUNE_BLOCK_LOAD_ALGORITHM == 0
+    constexpr auto load_alg = cub::BLOCK_LOAD_DIRECT;
+#  elif TUNE_BLOCK_LOAD_ALGORITHM == 1
+    constexpr auto load_alg = cub::BLOCK_LOAD_WARP_TRANSPOSE;
+#  elif TUNE_BLOCK_LOAD_ALGORITHM == 2
+    constexpr auto load_alg = cub::BLOCK_LOAD_VECTORIZE;
+#  endif
+    return cub::detail::batched_topk::batched_topk_policy{{{
+      cub::detail::batched_topk::worker_policy{TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, load_alg, store_alg},
+      cub::detail::batched_topk::worker_policy{TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, load_alg, store_alg},
+      cub::detail::batched_topk::worker_policy{TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, load_alg, store_alg},
+      cub::detail::batched_topk::worker_policy{TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, load_alg, store_alg},
+      cub::detail::batched_topk::worker_policy{TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, load_alg, store_alg},
+      cub::detail::batched_topk::worker_policy{TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, load_alg, store_alg},
+    }}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename KeyT, int MaxSegmentSize, int MaxNumSelected>
+void fixed_seg_size_topk_keys(
+  nvbench::state& state,
+  nvbench::type_list<KeyT, nvbench::enum_type<MaxSegmentSize>, nvbench::enum_type<MaxNumSelected>>)
+{
+  // Retrieve axis parameters
+  const auto max_elements      = static_cast<size_t>(state.get_int64("Elements{io}"));
+  const auto segment_size      = static_cast<::cuda::std::ptrdiff_t>(MaxSegmentSize);
+  const auto selected_elements = static_cast<::cuda::std::ptrdiff_t>(MaxNumSelected);
+  const auto num_segments      = ::cuda::std::max<std::size_t>(1, (max_elements / segment_size));
+  const auto elements          = num_segments * segment_size;
+  const auto total_num_items   = ::cuda::args::immediate{static_cast<::cuda::std::int64_t>(elements)};
+  const bit_entropy entropy    = str_to_entropy(state.get_string("Entropy"));
+
+  // Skip workloads where k exceeds the segment size
+  if (selected_elements >= segment_size)
+  {
+    state.skip("Skipping workload where K >= SegmentSize.");
+    return;
+  }
+
+  thrust::device_vector<KeyT> in_keys_buffer = generate(elements, entropy);
+  thrust::device_vector<KeyT> out_keys_buffer(selected_elements * num_segments, thrust::no_init);
+  auto d_keys_in_ptr  = thrust::raw_pointer_cast(in_keys_buffer.data());
+  auto d_keys_out_ptr = thrust::raw_pointer_cast(out_keys_buffer.data());
+  auto d_keys_in      = cuda::make_strided_iterator(cuda::make_counting_iterator(d_keys_in_ptr), segment_size);
+  auto d_keys_out     = cuda::make_strided_iterator(cuda::make_counting_iterator(d_keys_out_ptr), selected_elements);
+
+  auto segment_sizes    = ::cuda::args::constant<MaxSegmentSize>{};
+  auto k                = ::cuda::args::constant<MaxNumSelected>{};
+  auto select_direction = ::cuda::args::constant<cub::detail::topk::select::max>{};
+
+  state.add_element_count(elements, "NumElements");
+  state.add_element_count(segment_size, "SegmentSize");
+  state.add_element_count(selected_elements, "NumSelectedElements");
+  state.add_global_memory_reads<KeyT>(elements, "InputKeys");
+  state.add_global_memory_writes<KeyT>(selected_elements * num_segments, "OutputKeys");
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(tuned_policy_selector{})
+#endif // !TUNE_BASE
+    );
+    // TODO(bgruber): call the public API once available
+    _CCCL_TRY_CUDA_API(
+      cub::detail::batched_topk::dispatch_with_env,
+      "batched topk failed",
+      d_keys_in,
+      d_keys_out,
+      static_cast<cub::NullType**>(nullptr),
+      static_cast<cub::NullType**>(nullptr),
+      segment_sizes,
+      k,
+      select_direction,
+      ::cuda::args::immediate{static_cast<::cuda::std::int64_t>(num_segments)},
+      total_num_items,
+      env);
+  });
+}
+
+using key_type_list          = nvbench::type_list<float>;
+using segment_size_type_list = nvbench::type_list<uint32_t>;
+using out_offset_type_list   = nvbench::type_list<uint32_t>;
+
+using segment_size_        = nvbench::type_list<uint32_t>;
+using out_offset_type_list = nvbench::type_list<uint32_t>;
+
+using small_segment_size_list = nvbench::enum_type_list<64, 128, 256, 512, 1024>;
+using small_k_list            = nvbench::enum_type_list<8, 16, 32, 128, 512, 1024>;
+
+NVBENCH_BENCH_TYPES(fixed_seg_size_topk_keys, NVBENCH_TYPE_AXES(key_type_list, small_segment_size_list, small_k_list))
+  .set_name("small")
+  .set_type_axes_names({"KeyT{ct}", "MaxSegmentSize{ct}", "MaxNumSelected{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(28, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.201", "0.000"});

cccl_upstream/cub/benchmarks/bench/segmented_topk/variable/common.cuh

@@ -0,0 +1,175 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#pragma once
+
+#include <thrust/device_vector.h>
+#include <thrust/tabulate.h>
+
+#include <cuda/random>
+#include <cuda/std/algorithm>
+#include <cuda/std/cmath>
+#include <cuda/std/cstdint>
+#include <cuda/std/random>
+
+#include <cstddef>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include <nvbench_helper.cuh>
+
+namespace
+{
+enum class pattern_kind : int
+{
+  random = 0,
+  quantized_random,
+  relu_quantized,
+  tie_heavy,
+  pivot_tie
+};
+
+[[nodiscard]] pattern_kind string_to_pattern(const std::string& pattern)
+{
+  if (pattern == "random")
+  {
+    return pattern_kind::random;
+  }
+  if (pattern == "quantized_random")
+  {
+    return pattern_kind::quantized_random;
+  }
+  if (pattern == "relu_quantized")
+  {
+    return pattern_kind::relu_quantized;
+  }
+  if (pattern == "tie_heavy")
+  {
+    return pattern_kind::tie_heavy;
+  }
+  if (pattern == "pivot_tie")
+  {
+    return pattern_kind::pivot_tie;
+  }
+  throw std::runtime_error("Invalid Pattern axis value: " + pattern);
+}
+
+template <int MaxSegmentSize, int K>
+[[nodiscard]] thrust::device_vector<float>
+gen_data(int num_segments, pattern_kind pattern, const cuda::std::int64_t* d_seg_sizes)
+{
+  const auto num_keys = static_cast<std::size_t>(num_segments) * static_cast<std::size_t>(MaxSegmentSize);
+  auto d_keys         = thrust::device_vector<float>{num_keys, thrust::no_init};
+
+  // gt_count == "greater-than count": number of 2.0 values placed at the tail of each segment's live region.
+  constexpr int gt_count = cuda::std::max(1, cuda::std::min(K / 4, MaxSegmentSize / 8));
+
+  thrust::tabulate(d_keys.begin(), d_keys.end(), [pattern, d_seg_sizes] __device__(std::size_t idx) -> float {
+    auto quantize = [](float base) -> float {
+      const auto r         = cuda::std::rint(base);
+      const auto scaled_fr = cuda::std::rint((base - r) * 32.0f);
+      return r + (scaled_fr / 32.0f);
+    };
+
+    auto random_value = [](unsigned long long idx) -> float {
+      cuda::pcg64 rng(42);
+      rng.discard(idx);
+      cuda::std::normal_distribution<float> normal(0.f, 1.f);
+      return normal(rng);
+    };
+
+    const auto j = static_cast<int>(idx % MaxSegmentSize);
+    switch (pattern)
+    {
+      //               ##
+      //              ####
+      //            ########
+      //          ############
+      //        ################
+      //     ######################
+      //  ##############################
+      //  ------------------------------
+      //  -3             0             3
+      case pattern_kind::random:
+        return random_value(idx);
+
+      //                |
+      //                |
+      //            |   |   |
+      //        |   |   |   |   |
+      //    |   |   |   |   |   |   |
+      //  ----------------------------
+      //  -3            0            3
+      case pattern_kind::quantized_random:
+        return quantize(random_value(idx));
+
+      //  |
+      //  |
+      //  |
+      //  |
+      //  |
+      //  |   |
+      //  |   |   |
+      //  |   |   |   |   |
+      //  |   |   |   |   |   |   |
+      //  ----------------------------
+      //  0                          3
+      case pattern_kind::relu_quantized:
+        return quantize(cuda::std::max(random_value(idx), 0.f));
+
+      //  |   |   |   |   |   |   |   |
+      //  |   |   |   |   |   |   |   |
+      //  |   |   |   |   |   |   |   |
+      //  --------------------------------
+      //  0/64                      63/64
+      case pattern_kind::tie_heavy:
+        return static_cast<float>(j % 64) / 64.f;
+
+      //  |
+      //  |
+      //  |
+      //  |
+      //  |
+      //  |
+      //  |
+      //  |                         |
+      //  ----------------------------
+      //  1.0                       2.0
+      case pattern_kind::pivot_tie: {
+        const auto seg_size = static_cast<int>(d_seg_sizes[idx / MaxSegmentSize]);
+        return (j >= seg_size - gt_count) ? 2.f : 1.f;
+      }
+      default:
+        _CCCL_UNREACHABLE();
+    }
+  });
+
+  return d_keys;
+}
+} // namespace
+
+const std::vector<std::string> valid_patterns = {
+  "random", "quantized_random", "relu_quantized", "tie_heavy", "pivot_tie"};
+
+using key_type_list = nvbench::type_list<float>;
+
+using max_segment_size_list = nvbench::enum_type_list< //
+  512,
+  1024,
+  2048,
+  4096,
+  8192
+#if 0 // need these, waiting for implementation to catch up
+  ,
+  16384,
+  32768,
+  65536,
+  131072,
+  262144,
+  524288,
+  1048576
+#endif
+  >;
+
+using k_list = nvbench::enum_type_list<512, 1024, 2048>;

cccl_upstream/cub/benchmarks/bench/segmented_topk/variable/indexed.cu

@@ -0,0 +1,99 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/dispatch/dispatch_batched_topk.cuh>
+
+#include <thrust/device_vector.h>
+#include <thrust/reduce.h>
+
+#include <cuda/argument>
+#include <cuda/iterator>
+#include <cuda/std/cstdint>
+
+#include <nvbench_helper.cuh>
+
+#include "common.cuh"
+
+// Indexed (arg-top-k) variant: each key carries a segment-local index as its value payload. The input values are
+// produced by a counting iterator that restarts at 0 for every segment, so indices are not (pre-)materialized in global
+// memory
+template <typename KeyT, typename IndexT, int MaxSegmentSize, int K>
+void decode_style_variable_topk_indexed(
+  nvbench::state& state, nvbench::type_list<KeyT, IndexT, nvbench::enum_type<MaxSegmentSize>, nvbench::enum_type<K>>)
+{
+  if constexpr (K > MaxSegmentSize)
+  {
+    state.skip("K > MaxSegmentSize.");
+    return;
+  }
+
+  const auto num_segments                                         = static_cast<int>(state.get_int64("NumSegments"));
+  const thrust::device_vector<cuda::std::int64_t> d_segment_sizes = generate(
+    static_cast<std::size_t>(num_segments),
+    bit_entropy::_1_000,
+    static_cast<cuda::std::int64_t>(K),
+    static_cast<cuda::std::int64_t>(MaxSegmentSize));
+  const auto input_elements  = thrust::reduce(d_segment_sizes.begin(), d_segment_sizes.end());
+  const auto output_elements = static_cast<std::size_t>(num_segments) * K;
+  const auto total_num_items = cuda::args::immediate{static_cast<cuda::std::int64_t>(input_elements)};
+
+  auto in_keys_buffer = gen_data<MaxSegmentSize, K>(
+    num_segments, string_to_pattern(state.get_string("Pattern")), thrust::raw_pointer_cast(d_segment_sizes.data()));
+  auto out_keys_buffer    = thrust::device_vector<KeyT>(output_elements, thrust::no_init);
+  auto out_indices_buffer = thrust::device_vector<IndexT>(output_elements, thrust::no_init);
+
+  auto segment_sizes_param = cuda::args::deferred_sequence{
+    thrust::raw_pointer_cast(d_segment_sizes.data()), cuda::args::bounds<1, MaxSegmentSize>()};
+  auto k_param            = cuda::args::constant<K>{};
+  auto select_direction   = cuda::args::constant<cub::detail::topk::select::max>{};
+  auto num_segments_param = cuda::args::immediate{static_cast<cuda::std::int64_t>(num_segments)};
+
+  auto d_keys_in = cuda::make_strided_iterator(
+    cuda::make_counting_iterator(thrust::raw_pointer_cast(in_keys_buffer.data())),
+    static_cast<cuda::std::ptrdiff_t>(MaxSegmentSize));
+  auto d_keys_out = cuda::make_strided_iterator(
+    cuda::make_counting_iterator(thrust::raw_pointer_cast(out_keys_buffer.data())),
+    static_cast<cuda::std::ptrdiff_t>(K));
+
+  // Input values: every segment maps to the same counting iterator starting at 0, so values are segment-local indices.
+  auto d_indices_in  = cuda::make_constant_iterator(cuda::make_counting_iterator(IndexT{0}));
+  auto d_indices_out = cuda::make_strided_iterator(
+    cuda::make_counting_iterator(thrust::raw_pointer_cast(out_indices_buffer.data())),
+    static_cast<cuda::std::ptrdiff_t>(K));
+
+  state.add_element_count(input_elements, "NumElements");
+  state.add_global_memory_reads<KeyT>(input_elements, "InputKeys");
+  state.add_global_memory_reads<cuda::std::int64_t>(num_segments, "SegmentSizes");
+  state.add_global_memory_writes<KeyT>(output_elements, "OutputKeys");
+  state.add_global_memory_writes<IndexT>(output_elements, "OutputIndices");
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(alloc, launch);
+    // TODO(bgruber): call the public API once available
+    _CCCL_TRY_CUDA_API(
+      cub::detail::batched_topk::dispatch_with_env,
+      "batched topk failed",
+      d_keys_in,
+      d_keys_out,
+      d_indices_in,
+      d_indices_out,
+      segment_sizes_param,
+      k_param,
+      select_direction,
+      num_segments_param,
+      total_num_items,
+      env);
+  });
+}
+
+// Index type is a compile-time axis: i32 for now, extensible to i64.
+using index_type_list = nvbench::type_list<cuda::std::int32_t>;
+
+NVBENCH_BENCH_TYPES(decode_style_variable_topk_indexed,
+                    NVBENCH_TYPE_AXES(key_type_list, index_type_list, max_segment_size_list, k_list))
+  .set_name("decode_style_variable_topk_indexed")
+  .set_type_axes_names({"KeyT{ct}", "IndexT{ct}", "MaxSegmentSize{ct}", "K{ct}"})
+  .add_int64_axis("NumSegments", {1, 2, 4, 8, 16, 32})
+  .add_string_axis("Pattern", valid_patterns);

cccl_upstream/cub/benchmarks/bench/segmented_topk/variable/keys.cu

@@ -0,0 +1,83 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/detail/choose_offset.cuh>
+#include <cub/device/dispatch/dispatch_batched_topk.cuh>
+
+#include <thrust/device_vector.h>
+#include <thrust/reduce.h>
+
+#include <cuda/argument>
+#include <cuda/iterator>
+
+#include <nvbench_helper.cuh>
+
+#include "common.cuh"
+
+template <typename KeyT, int MaxSegmentSize, int K>
+void decode_style_variable_topk_keys(
+  nvbench::state& state, nvbench::type_list<KeyT, nvbench::enum_type<MaxSegmentSize>, nvbench::enum_type<K>>)
+{
+  if constexpr (K > MaxSegmentSize)
+  {
+    state.skip("K > MaxSegmentSize.");
+    return;
+  }
+
+  const auto num_segments                                         = static_cast<int>(state.get_int64("NumSegments"));
+  const thrust::device_vector<cuda::std::int64_t> d_segment_sizes = generate(
+    static_cast<std::size_t>(num_segments),
+    bit_entropy::_1_000,
+    static_cast<cuda::std::int64_t>(K),
+    static_cast<cuda::std::int64_t>(MaxSegmentSize));
+  const auto input_elements  = thrust::reduce(d_segment_sizes.begin(), d_segment_sizes.end());
+  const auto output_elements = static_cast<std::size_t>(num_segments) * K;
+  const auto total_num_items = cuda::args::immediate{static_cast<cuda::std::int64_t>(input_elements)};
+
+  auto in_keys_buffer = gen_data<MaxSegmentSize, K>(
+    num_segments, string_to_pattern(state.get_string("Pattern")), thrust::raw_pointer_cast(d_segment_sizes.data()));
+  auto out_keys_buffer = thrust::device_vector<KeyT>(output_elements, thrust::no_init);
+
+  auto segment_sizes_param = cuda::args::deferred_sequence{
+    thrust::raw_pointer_cast(d_segment_sizes.data()), cuda::args::bounds<1, MaxSegmentSize>()};
+  auto k_param            = cuda::args::constant<K>{};
+  auto select_direction   = cuda::args::constant<cub::detail::topk::select::max>{};
+  auto num_segments_param = cuda::args::immediate{static_cast<cuda::std::int64_t>(num_segments)};
+
+  auto d_keys_in = cuda::make_strided_iterator(
+    cuda::make_counting_iterator(thrust::raw_pointer_cast(in_keys_buffer.data())),
+    static_cast<cuda::std::ptrdiff_t>(MaxSegmentSize));
+  auto d_keys_out = cuda::make_strided_iterator(
+    cuda::make_counting_iterator(thrust::raw_pointer_cast(out_keys_buffer.data())),
+    static_cast<cuda::std::ptrdiff_t>(K));
+
+  state.add_element_count(input_elements, "NumElements");
+  state.add_global_memory_reads<KeyT>(input_elements, "InputKeys");
+  state.add_global_memory_reads<cuda::std::int64_t>(num_segments, "SegmentSizes");
+  state.add_global_memory_writes<KeyT>(output_elements, "OutputKeys");
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(alloc, launch);
+    // TODO(bgruber): call the public API once available
+    _CCCL_TRY_CUDA_API(
+      cub::detail::batched_topk::dispatch_with_env,
+      "batched topk failed",
+      d_keys_in,
+      d_keys_out,
+      static_cast<cub::NullType**>(nullptr),
+      static_cast<cub::NullType**>(nullptr),
+      segment_sizes_param,
+      k_param,
+      select_direction,
+      num_segments_param,
+      total_num_items,
+      env);
+  });
+}
+
+NVBENCH_BENCH_TYPES(decode_style_variable_topk_keys, NVBENCH_TYPE_AXES(key_type_list, max_segment_size_list, k_list))
+  .set_name("decode_style_variable_topk_keys")
+  .set_type_axes_names({"KeyT{ct}", "MaxSegmentSize{ct}", "K{ct}"})
+  .add_int64_axis("NumSegments", {1, 2, 4, 8, 16, 32})
+  .add_string_axis("Pattern", valid_patterns);

cccl_upstream/cub/benchmarks/bench/select/flagged.cu

@@ -0,0 +1,116 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_select.cuh>
+
+#include <thrust/count.h>
+
+#include <cuda/std/algorithm>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+template <typename InputT>
+struct bench_policy_selector
+{
+  [[nodiscard]] _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::SelectPolicy
+  {
+    return {cub::SelectAlgorithm::lookback,
+            {TUNE_THREADS_PER_BLOCK,
+             TUNE_ITEMS_PER_THREAD,
+             (TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE),
+             (TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : cub::LOAD_CA),
+             cub::BLOCK_SCAN_WARP_SCANS,
+             lookback_delay_policy}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T, typename InPlace>
+void select(nvbench::state& state, nvbench::type_list<T, InPlace>)
+{
+  using offset_t = int64_t;
+
+  // Retrieve axis parameters
+  const auto elements       = state.get_int64("Elements{io}");
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  auto generator = generate(elements, entropy);
+
+  thrust::device_vector<T> in       = generator;
+  thrust::device_vector<bool> flags = generator;
+  thrust::device_vector<offset_t> num_selected(1);
+
+  // TODO Extract into helper TU
+  const auto selected_elements = thrust::count(flags.cbegin(), flags.cend(), true);
+  thrust::device_vector<T> out(selected_elements, thrust::no_init);
+
+  T* d_in                  = thrust::raw_pointer_cast(in.data());
+  T* d_out                 = thrust::raw_pointer_cast(out.data());
+  const bool* d_flags      = thrust::raw_pointer_cast(flags.data());
+  offset_t* d_num_selected = thrust::raw_pointer_cast(num_selected.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_reads<bool>(elements);
+  state.add_global_memory_writes<T>(selected_elements);
+  state.add_global_memory_writes<offset_t>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<T>{})
+#endif // !TUNE_BASE
+    );
+    if constexpr (InPlace::value)
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSelect::Flagged,
+        "DeviceSelect::Flagged failed",
+        d_in,
+        d_flags,
+        d_num_selected,
+        static_cast<offset_t>(elements),
+        env);
+    }
+    else
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSelect::Flagged,
+        "DeviceSelect::Flagged failed",
+        static_cast<const T*>(d_in),
+        d_flags,
+        d_out,
+        d_num_selected,
+        static_cast<offset_t>(elements),
+        env);
+    }
+  });
+}
+
+using ::cuda::std::false_type;
+using ::cuda::std::true_type;
+#ifdef TUNE_InPlace
+using is_in_place = nvbench::type_list<TUNE_InPlace>; // expands to "false_type" or "true_type"
+#else // !defined(TUNE_InPlace)
+using is_in_place = nvbench::type_list<false_type, true_type>;
+#endif // TUNE_InPlace
+
+NVBENCH_BENCH_TYPES(select, NVBENCH_TYPE_AXES(fundamental_types, is_in_place))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "InPlace{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.000"});

cccl_upstream/cub/benchmarks/bench/select/if.cu

@@ -0,0 +1,117 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_select.cuh>
+
+#include <thrust/count.h>
+
+#include <cuda/std/algorithm>
+
+#include <limits>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+template <typename InputT>
+struct bench_policy_selector
+{
+  [[nodiscard]] _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::SelectPolicy
+  {
+    return {cub::SelectAlgorithm::lookback,
+            {TUNE_THREADS_PER_BLOCK,
+             TUNE_ITEMS_PER_THREAD,
+             (TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE),
+             (TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : cub::LOAD_CA),
+             cub::BLOCK_SCAN_WARP_SCANS,
+             lookback_delay_policy}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T, typename InPlace>
+void select(nvbench::state& state, nvbench::type_list<T, InPlace>)
+{
+  using offset_t    = int64_t;
+  using select_op_t = less_then_t<T>;
+
+  // Retrieve axis parameters
+  const auto elements       = state.get_int64("Elements{io}");
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  const T val = lerp_min_max<T>(entropy_to_probability(entropy));
+  select_op_t select_op{val};
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<offset_t> num_selected(1);
+
+  // TODO Extract into helper TU
+  const auto selected_elements = thrust::count_if(in.cbegin(), in.cend(), select_op);
+  thrust::device_vector<T> out(selected_elements, thrust::no_init);
+
+  T* d_in                  = thrust::raw_pointer_cast(in.data());
+  T* d_out                 = thrust::raw_pointer_cast(out.data());
+  offset_t* d_num_selected = thrust::raw_pointer_cast(num_selected.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(selected_elements);
+  state.add_global_memory_writes<offset_t>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<T>{})
+#endif // !TUNE_BASE
+    );
+    if constexpr (InPlace::value)
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSelect::If,
+        "select_if failed",
+        d_in,
+        d_num_selected,
+        static_cast<offset_t>(elements),
+        select_op,
+        env);
+    }
+    else
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSelect::If,
+        "select_if failed",
+        static_cast<const T*>(d_in),
+        d_out,
+        d_num_selected,
+        static_cast<offset_t>(elements),
+        select_op,
+        env);
+    }
+  });
+}
+
+using ::cuda::std::false_type;
+using ::cuda::std::true_type;
+#ifdef TUNE_InPlace
+using is_in_place = nvbench::type_list<TUNE_InPlace>; // expands to "false_type" or "true_type"
+#else // !defined(TUNE_InPlace)
+using is_in_place = nvbench::type_list<false_type, true_type>;
+#endif // TUNE_InPlace
+
+NVBENCH_BENCH_TYPES(select, NVBENCH_TYPE_AXES(fundamental_types, is_in_place))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "InPlace{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.000"});

cccl_upstream/cub/benchmarks/bench/select/unique.cu

@@ -0,0 +1,120 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include <cub/device/device_select.cuh>
+
+#include <cuda/std/algorithm>
+
+#include <limits>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+template <typename InputT>
+struct bench_policy_selector
+{
+  [[nodiscard]] _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::SelectPolicy
+  {
+    return {cub::SelectAlgorithm::lookback,
+            {TUNE_THREADS_PER_BLOCK,
+             TUNE_ITEMS_PER_THREAD,
+             (TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE),
+             (TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : cub::LOAD_CA),
+             cub::BLOCK_SCAN_WARP_SCANS,
+             lookback_delay_policy}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T, typename InPlace>
+static void unique(nvbench::state& state, nvbench::type_list<T, InPlace>)
+{
+  using offset_t = int64_t;
+
+  // Retrieve axis parameters
+  const auto elements         = state.get_int64("Elements{io}");
+  const auto max_segment_size = state.get_int64("MaxSegSize");
+
+  thrust::device_vector<T> in = generate.uniform.key_segments(elements, /* min_segmented_size */ 1, max_segment_size);
+  thrust::device_vector<T> out(elements, thrust::no_init);
+  thrust::device_vector<offset_t> num_unique_out(1);
+
+  T* d_in                = thrust::raw_pointer_cast(in.data());
+  T* d_out               = thrust::raw_pointer_cast(out.data());
+  offset_t* d_num_unique = thrust::raw_pointer_cast(num_unique_out.data());
+
+  // Get number of unique elements for metrics
+  _CCCL_TRY_CUDA_API(
+    cub::DeviceSelect::Unique,
+    "select_unique failed",
+    d_in,
+    d_out,
+    d_num_unique,
+    static_cast<offset_t>(elements),
+    ::cuda::std::equal_to<>{});
+  cudaDeviceSynchronize();
+  const offset_t num_unique = num_unique_out[0];
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(num_unique);
+  state.add_global_memory_writes<offset_t>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<T>{})
+#endif // !TUNE_BASE
+    );
+    if constexpr (InPlace::value)
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSelect::Unique,
+        "select_unique failed",
+        d_in,
+        d_num_unique,
+        static_cast<offset_t>(elements),
+        ::cuda::std::equal_to<>{},
+        env);
+    }
+    else
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSelect::Unique,
+        "select_unique failed",
+        d_in,
+        d_out,
+        d_num_unique,
+        static_cast<offset_t>(elements),
+        ::cuda::std::equal_to<>{},
+        env);
+    }
+  });
+}
+
+using ::cuda::std::false_type;
+using ::cuda::std::true_type;
+#ifdef TUNE_InPlace
+using is_in_place = nvbench::type_list<TUNE_InPlace>; // expands to "false_type" or "true_type"
+#else // !defined(TUNE_InPlace)
+using is_in_place = nvbench::type_list<false_type, true_type>;
+#endif // TUNE_InPlace
+
+NVBENCH_BENCH_TYPES(unique, NVBENCH_TYPE_AXES(fundamental_types, is_in_place))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "InPlace{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegSize", {1, 4, 8});

cccl_upstream/cub/benchmarks/bench/select/unique_by_key.cu

@@ -0,0 +1,139 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_select.cuh>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  else // TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+
+struct bench_unique_by_key_policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::UniqueByKeyPolicy
+  {
+    return {TUNE_THREADS,
+            TUNE_ITEMS,
+            TUNE_LOAD_ALGORITHM,
+            TUNE_LOAD_MODIFIER,
+            cub::BLOCK_SCAN_WARP_SCANS,
+            lookback_delay_policy};
+  }
+};
+#endif // !TUNE_BASE
+
+template <class KeyT, class ValueT, class OffsetT>
+static void select(nvbench::state& state, nvbench::type_list<KeyT, ValueT, OffsetT>)
+{
+  using equality_op_t = cuda::std::equal_to<>;
+
+  const auto elements                    = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  constexpr std::size_t min_segment_size = 1;
+  const std::size_t max_segment_size     = static_cast<std::size_t>(state.get_int64("MaxSegSize"));
+
+  thrust::device_vector<OffsetT> num_runs_out(1);
+  thrust::device_vector<ValueT> in_vals(elements);
+  thrust::device_vector<ValueT> out_vals(elements);
+  thrust::device_vector<KeyT> out_keys(elements);
+  thrust::device_vector<KeyT> in_keys = generate.uniform.key_segments(elements, min_segment_size, max_segment_size);
+
+  const KeyT* d_in_keys   = thrust::raw_pointer_cast(in_keys.data());
+  KeyT* d_out_keys        = thrust::raw_pointer_cast(out_keys.data());
+  const ValueT* d_in_vals = thrust::raw_pointer_cast(in_vals.data());
+  ValueT* d_out_vals      = thrust::raw_pointer_cast(out_vals.data());
+  OffsetT* d_num_runs_out = thrust::raw_pointer_cast(num_runs_out.data());
+
+  const auto num_items = static_cast<OffsetT>(elements);
+
+  // Pre-computation to get num_runs for statistics
+  _CCCL_TRY_CUDA_API(
+    cub::DeviceSelect::UniqueByKey,
+    "UniqueByKey failed",
+    d_in_keys,
+    d_in_vals,
+    d_out_keys,
+    d_out_vals,
+    d_num_runs_out,
+    num_items,
+    equality_op_t{});
+  _CCCL_TRY_CUDA_API(cudaDeviceSynchronize, "Sync failed");
+  const OffsetT num_runs = num_runs_out[0];
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<ValueT>(num_runs);
+  state.add_global_memory_writes<KeyT>(num_runs);
+  state.add_global_memory_writes<OffsetT>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_unique_by_key_policy_selector{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceSelect::UniqueByKey,
+      "UniqueByKey failed",
+      d_in_keys,
+      d_in_vals,
+      d_out_keys,
+      d_out_vals,
+      d_num_runs_out,
+      num_items,
+      equality_op_t{},
+      env);
+  });
+}
+
+using some_offset_types = nvbench::type_list<nvbench::int32_t>;
+
+#ifdef TUNE_KeyT
+using key_types = nvbench::type_list<TUNE_KeyT>;
+#else // !defined(TUNE_KeyT)
+using key_types =
+  nvbench::type_list<int8_t,
+                     int16_t,
+                     int32_t,
+                     int64_t
+#  if _CCCL_HAS_INT128()
+                     ,
+                     int128_t
+#  endif
+                     >;
+#endif // TUNE_KeyT
+
+#ifdef TUNE_ValueT
+using value_types = nvbench::type_list<TUNE_ValueT>;
+#else // !defined(TUNE_ValueT)
+using value_types = all_types;
+#endif // TUNE_ValueT
+
+NVBENCH_BENCH_TYPES(select, NVBENCH_TYPE_AXES(key_types, value_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegSize", {1, 4, 8});

cccl_upstream/cub/benchmarks/bench/topk/keys.cu

@@ -0,0 +1,111 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/device/device_topk.cuh>
+
+#include <cuda/__execution/determinism.h>
+#include <cuda/__execution/output_ordering.h>
+#include <cuda/__execution/require.h>
+#include <cuda/__execution/tune.h>
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 1:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_BLOCK_LOAD_ALGORITHM ld 0:2:1
+
+#if !TUNE_BASE
+template <class KeyInT>
+struct policy_selector_t
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const
+    -> cub::detail::topk::topk_policy
+  {
+#  if TUNE_BLOCK_LOAD_ALGORITHM == 0
+    constexpr auto load_alg = cub::BLOCK_LOAD_DIRECT;
+#  elif TUNE_BLOCK_LOAD_ALGORITHM == 1
+    constexpr auto load_alg = cub::BLOCK_LOAD_WARP_TRANSPOSE;
+#  elif TUNE_BLOCK_LOAD_ALGORITHM == 2
+    constexpr auto load_alg = cub::BLOCK_LOAD_VECTORIZE;
+#  endif
+
+    constexpr int nominal_4b_items_per_thread = TUNE_ITEMS_PER_THREAD;
+    constexpr int items_per_thread            = cuda::std::max(1, (nominal_4b_items_per_thread * 4 / sizeof(KeyInT)));
+    return cub::detail::topk::topk_policy{
+      TUNE_THREADS_PER_BLOCK,
+      items_per_thread,
+      load_alg,
+      cub::BLOCK_SCAN_WARP_SCANS,
+      cub::detail::topk::calc_bits_per_pass<KeyInT>()};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename KeyT, typename OffsetT, typename OutOffsetT>
+void topk_keys(nvbench::state& state, nvbench::type_list<KeyT, OffsetT, OutOffsetT>)
+{
+  // Retrieve axis parameters
+  const auto elements          = static_cast<size_t>(state.get_int64("Elements{io}"));
+  const auto selected_elements = static_cast<size_t>(state.get_int64("SelectedElements"));
+  const bit_entropy entropy    = str_to_entropy(state.get_string("Entropy"));
+
+  // Skip benchmarks at runtime
+  if (selected_elements >= elements)
+  {
+    state.skip("We only support the case where the variable SelectedElements is smaller than the variable "
+               "Elements{io}.");
+    return;
+  }
+
+  thrust::device_vector<KeyT> in_keys = generate(elements, entropy);
+  thrust::device_vector<KeyT> out_keys(selected_elements, thrust::no_init);
+  const KeyT* d_keys_in = thrust::raw_pointer_cast(in_keys.data());
+  KeyT* d_keys_out      = thrust::raw_pointer_cast(out_keys.data());
+
+  state.add_element_count(elements, "NumElements");
+  state.add_element_count(selected_elements, "NumSelectedElements");
+  state.add_global_memory_reads<KeyT>(elements, "InputKeys");
+  state.add_global_memory_writes<KeyT>(selected_elements, "OutputKeys");
+
+  // TODO(bgruber): call cub::DeviceTopK::MaxKeys with a the caching_allocator_t once we have an env-overload without
+  // temporary storage
+  auto env = cuda::std::execution::env{
+    cuda::execution::require(cuda::execution::determinism::not_guaranteed, cuda::execution::output_ordering::unsorted)
+#if !TUNE_BASE
+      ,
+    cuda::execution::tune(policy_selector_t<KeyT>{})
+#endif // !TUNE_BASE
+  };
+
+  // Allocate temporary storage
+  size_t temp_size{};
+  cub::DeviceTopK::MaxKeys(
+    nullptr,
+    temp_size,
+    d_keys_in,
+    d_keys_out,
+    static_cast<OffsetT>(elements),
+    static_cast<OutOffsetT>(selected_elements),
+    env);
+  thrust::device_vector<nvbench::uint8_t> temp(temp_size, thrust::no_init);
+  auto* temp_storage = thrust::raw_pointer_cast(temp.data());
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env_with_stream = cuda::std::execution::env{cuda::stream_ref{launch.get_stream().get_stream()}, env};
+    cub::DeviceTopK::MaxKeys(
+      temp_storage,
+      temp_size,
+      d_keys_in,
+      d_keys_out,
+      static_cast<OffsetT>(elements),
+      static_cast<OutOffsetT>(selected_elements),
+      env_with_stream);
+  });
+}
+
+NVBENCH_BENCH_TYPES(topk_keys, NVBENCH_TYPE_AXES(fundamental_types, offset_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "OffsetT{ct}", "OutOffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("SelectedElements", nvbench::range(3, 23, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.201", "0.000"});

cccl_upstream/cub/benchmarks/bench/topk/pairs.cu

@@ -0,0 +1,120 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cub/device/device_topk.cuh>
+
+#include <cuda/__execution/determinism.h>
+#include <cuda/__execution/output_ordering.h>
+#include <cuda/__execution/require.h>
+#include <cuda/__execution/tune.h>
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 1:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_BLOCK_LOAD_ALGORITHM ld 0:2:1
+
+#if !TUNE_BASE
+template <class KeyInT>
+struct policy_selector_t
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const
+    -> cub::detail::topk::topk_policy
+  {
+#  if TUNE_BLOCK_LOAD_ALGORITHM == 0
+    constexpr auto load_alg = cub::BLOCK_LOAD_DIRECT;
+#  elif TUNE_BLOCK_LOAD_ALGORITHM == 1
+    constexpr auto load_alg = cub::BLOCK_LOAD_WARP_TRANSPOSE;
+#  elif TUNE_BLOCK_LOAD_ALGORITHM == 2
+    constexpr auto load_alg = cub::BLOCK_LOAD_VECTORIZE;
+#  endif
+
+    constexpr int nominal_4b_items_per_thread = TUNE_ITEMS_PER_THREAD;
+    constexpr int items_per_thread            = cuda::std::max(1, (nominal_4b_items_per_thread * 4 / sizeof(KeyInT)));
+    return cub::detail::topk::topk_policy{
+      TUNE_THREADS_PER_BLOCK,
+      items_per_thread,
+      load_alg,
+      cub::BLOCK_SCAN_WARP_SCANS,
+      cub::detail::topk::calc_bits_per_pass<KeyInT>()};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename KeyT, typename ValueT, typename OffsetT, typename OutOffsetT>
+void topk_pairs(nvbench::state& state, nvbench::type_list<KeyT, ValueT, OffsetT, OutOffsetT>)
+{
+  // Retrieve axis parameters
+  const auto elements          = static_cast<size_t>(state.get_int64("Elements{io}"));
+  const auto selected_elements = static_cast<size_t>(state.get_int64("SelectedElements"));
+  const bit_entropy entropy    = str_to_entropy(state.get_string("Entropy"));
+
+  // Skip benchmarks at runtime
+  if (selected_elements >= elements)
+  {
+    state.skip("We only support the case where the variable SelectedElements is smaller than the variable "
+               "Elements{io}.");
+    return;
+  }
+
+  thrust::device_vector<KeyT> in_keys     = generate(elements, entropy);
+  thrust::device_vector<ValueT> in_values = generate(elements);
+  thrust::device_vector<KeyT> out_keys(selected_elements, thrust::no_init);
+  thrust::device_vector<ValueT> out_values(selected_elements, thrust::no_init);
+
+  const KeyT* d_keys_in     = thrust::raw_pointer_cast(in_keys.data());
+  KeyT* d_keys_out          = thrust::raw_pointer_cast(out_keys.data());
+  const ValueT* d_values_in = thrust::raw_pointer_cast(in_values.data());
+  ValueT* d_values_out      = thrust::raw_pointer_cast(out_values.data());
+
+  state.add_element_count(elements, "NumElements");
+  state.add_element_count(selected_elements, "NumSelectedElements");
+  state.add_global_memory_reads<KeyT>(elements, "InputKeys");
+  state.add_global_memory_reads<ValueT>(elements, "InputValues");
+  state.add_global_memory_writes<KeyT>(selected_elements, "OutputKeys");
+  state.add_global_memory_writes<ValueT>(selected_elements, "OutputVales");
+
+  auto env = cuda::std::execution::env{
+    cuda::execution::require(cuda::execution::determinism::not_guaranteed, cuda::execution::output_ordering::unsorted)
+#if !TUNE_BASE
+      ,
+    cuda::execution::tune(policy_selector_t<KeyT>{})
+#endif // !TUNE_BASE
+  };
+
+  // Allocate temporary storage
+  size_t temp_size{};
+  cub::DeviceTopK::MaxPairs(
+    nullptr,
+    temp_size,
+    d_keys_in,
+    d_keys_out,
+    d_values_in,
+    d_values_out,
+    static_cast<OffsetT>(elements),
+    static_cast<OutOffsetT>(selected_elements),
+    env);
+  thrust::device_vector<nvbench::uint8_t> temp(temp_size, thrust::no_init);
+  auto* temp_storage = thrust::raw_pointer_cast(temp.data());
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env_with_stream = cuda::std::execution::env{cuda::stream_ref{launch.get_stream().get_stream()}, env};
+    cub::DeviceTopK::MaxPairs(
+      temp_storage,
+      temp_size,
+      d_keys_in,
+      d_keys_out,
+      d_values_in,
+      d_values_out,
+      static_cast<OffsetT>(elements),
+      static_cast<OutOffsetT>(selected_elements),
+      env_with_stream);
+  });
+}
+
+NVBENCH_BENCH_TYPES(topk_pairs, NVBENCH_TYPE_AXES(integral_types, integral_types, offset_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}", "OffsetT{ct}", "OutOffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("SelectedElements", nvbench::range(3, 23, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.201", "0.000"});

cccl_upstream/cub/benchmarks/bench/transform/applications/P1/bfloat16.h

@@ -0,0 +1,235 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#pragma once
+
+#include <cuda_bf16.h>
+
+#include <cmath>
+#include <cstdint>
+#include <cstring>
+
+#include <nvbench/type_strings.cuh>
+
+// ============================================================================
+// BFloat16 type — replicated from c10::BFloat16
+// (torch/headeronly/util/BFloat16.h)
+// ============================================================================
+
+namespace bf16_detail
+{
+inline __host__ __device__ float f32_from_bits(uint16_t src)
+{
+  float res    = 0;
+  uint32_t tmp = src;
+  tmp <<= 16;
+  std::memcpy(&res, &tmp, sizeof(tmp));
+  return res;
+}
+
+inline __host__ __device__ uint16_t round_to_nearest_even(float src)
+{
+  if (std::isnan(src))
+  {
+    return UINT16_C(0x7FC0);
+  }
+  else
+  {
+    uint32_t U32;
+    std::memcpy(&U32, &src, sizeof(U32));
+    uint32_t rounding_bias = ((U32 >> 16) & 1) + UINT32_C(0x7FFF);
+    return static_cast<uint16_t>((U32 + rounding_bias) >> 16);
+  }
+}
+} // namespace bf16_detail
+
+struct alignas(2) BFloat16
+{
+  uint16_t x;
+
+  BFloat16() = default;
+
+  struct from_bits_t
+  {};
+  static constexpr __host__ __device__ from_bits_t from_bits()
+  {
+    return from_bits_t();
+  }
+
+  constexpr __host__ __device__ BFloat16(unsigned short bits, from_bits_t)
+      : x(bits)
+  {}
+
+  /* implicit */ inline __host__ __device__ BFloat16(float value);
+  inline __host__ __device__ operator float() const;
+
+  inline __host__ __device__ BFloat16(const __nv_bfloat16& value);
+  explicit inline __host__ __device__ operator __nv_bfloat16() const;
+};
+
+inline __host__ __device__ BFloat16::BFloat16(float value)
+{
+  NV_IF_ELSE_TARGET(NV_PROVIDES_SM_80,
+                    ({
+                      __nv_bfloat16 tmp = __float2bfloat16(value);
+                      x                 = *reinterpret_cast<const unsigned short*>(&tmp);
+                    }),
+                    ({ x = bf16_detail::round_to_nearest_even(value); }));
+}
+
+inline __host__ __device__ BFloat16::operator float() const
+{
+  return __bfloat162float(*reinterpret_cast<const __nv_bfloat16*>(&x));
+}
+
+inline __host__ __device__ BFloat16::BFloat16(const __nv_bfloat16& value)
+{
+  x = *reinterpret_cast<const unsigned short*>(&value);
+}
+inline __host__ __device__ BFloat16::operator __nv_bfloat16() const
+{
+  return *reinterpret_cast<const __nv_bfloat16*>(&x);
+}
+
+// Arithmetic — BFloat16 x BFloat16 → BFloat16
+inline __host__ __device__ BFloat16 operator+(const BFloat16& a, const BFloat16& b)
+{
+  return static_cast<float>(a) + static_cast<float>(b);
+}
+inline __host__ __device__ BFloat16 operator-(const BFloat16& a, const BFloat16& b)
+{
+  return static_cast<float>(a) - static_cast<float>(b);
+}
+inline __host__ __device__ BFloat16 operator*(const BFloat16& a, const BFloat16& b)
+{
+  return static_cast<float>(a) * static_cast<float>(b);
+}
+inline __host__ __device__ BFloat16 operator/(const BFloat16& a, const BFloat16& b)
+{
+  return static_cast<float>(a) / static_cast<float>(b);
+}
+inline __host__ __device__ BFloat16 operator-(const BFloat16& a)
+{
+  return -static_cast<float>(a);
+}
+
+// Compound assignment — BFloat16
+inline __host__ __device__ BFloat16& operator+=(BFloat16& a, const BFloat16& b)
+{
+  a = a + b;
+  return a;
+}
+inline __host__ __device__ BFloat16& operator-=(BFloat16& a, const BFloat16& b)
+{
+  a = a - b;
+  return a;
+}
+inline __host__ __device__ BFloat16& operator*=(BFloat16& a, const BFloat16& b)
+{
+  a = a * b;
+  return a;
+}
+inline __host__ __device__ BFloat16& operator/=(BFloat16& a, const BFloat16& b)
+{
+  a = a / b;
+  return a;
+}
+
+// Arithmetic — BFloat16 x float → float
+inline __host__ __device__ float operator+(BFloat16 a, float b)
+{
+  return static_cast<float>(a) + b;
+}
+inline __host__ __device__ float operator-(BFloat16 a, float b)
+{
+  return static_cast<float>(a) - b;
+}
+inline __host__ __device__ float operator*(BFloat16 a, float b)
+{
+  return static_cast<float>(a) * b;
+}
+inline __host__ __device__ float operator/(BFloat16 a, float b)
+{
+  return static_cast<float>(a) / b;
+}
+inline __host__ __device__ float operator+(float a, BFloat16 b)
+{
+  return a + static_cast<float>(b);
+}
+inline __host__ __device__ float operator-(float a, BFloat16 b)
+{
+  return a - static_cast<float>(b);
+}
+inline __host__ __device__ float operator*(float a, BFloat16 b)
+{
+  return a * static_cast<float>(b);
+}
+inline __host__ __device__ float operator/(float a, BFloat16 b)
+{
+  return a / static_cast<float>(b);
+}
+
+// Compound assignment — float x BFloat16 → float
+inline __host__ __device__ float& operator+=(float& a, const BFloat16& b)
+{
+  return a += static_cast<float>(b);
+}
+inline __host__ __device__ float& operator-=(float& a, const BFloat16& b)
+{
+  return a -= static_cast<float>(b);
+}
+inline __host__ __device__ float& operator*=(float& a, const BFloat16& b)
+{
+  return a *= static_cast<float>(b);
+}
+inline __host__ __device__ float& operator/=(float& a, const BFloat16& b)
+{
+  return a /= static_cast<float>(b);
+}
+
+// Arithmetic — BFloat16 x int → BFloat16
+inline __host__ __device__ BFloat16 operator+(BFloat16 a, int b)
+{
+  return a + static_cast<BFloat16>(static_cast<float>(b));
+}
+inline __host__ __device__ BFloat16 operator-(BFloat16 a, int b)
+{
+  return a - static_cast<BFloat16>(static_cast<float>(b));
+}
+inline __host__ __device__ BFloat16 operator*(BFloat16 a, int b)
+{
+  return a * static_cast<BFloat16>(static_cast<float>(b));
+}
+inline __host__ __device__ BFloat16 operator/(BFloat16 a, int b)
+{
+  return a / static_cast<BFloat16>(static_cast<float>(b));
+}
+inline __host__ __device__ BFloat16 operator+(int a, BFloat16 b)
+{
+  return static_cast<BFloat16>(static_cast<float>(a)) + b;
+}
+inline __host__ __device__ BFloat16 operator-(int a, BFloat16 b)
+{
+  return static_cast<BFloat16>(static_cast<float>(a)) - b;
+}
+inline __host__ __device__ BFloat16 operator*(int a, BFloat16 b)
+{
+  return static_cast<BFloat16>(static_cast<float>(a)) * b;
+}
+inline __host__ __device__ BFloat16 operator/(int a, BFloat16 b)
+{
+  return static_cast<BFloat16>(static_cast<float>(a)) / b;
+}
+
+// Comparison — for std::min/std::max
+inline __host__ __device__ bool operator>(BFloat16& lhs, BFloat16& rhs)
+{
+  return float(lhs) > float(rhs);
+}
+inline __host__ __device__ bool operator<(BFloat16& lhs, BFloat16& rhs)
+{
+  return float(lhs) < float(rhs);
+}
+
+// NVBench type registration
+NVBENCH_DECLARE_TYPE_STRINGS(BFloat16, "bf16", "BFloat16");

cccl_upstream/cub/benchmarks/bench/transform/applications/P1/pytorch.cu

@@ -0,0 +1,987 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// ============================================================================
+// NVBench benchmarks for chained elementwise operations was put together by Matthias Jouanneaux (DevTech). It mimics
+// how pytorch uses element-wise kernels (i.e. cub::DeviceTransform) and also tries to preserve pytorch's operators and
+// utility types. The main difference to ordinary CCCL benchmarks is the chaining of several operations in the
+// benchmark's critical section. Furthermore, there are 4 types of work loads covering the combinations of few vs. many
+// input buffers and few vs. many instructions in the kernel. For more discussion see:
+// https://github.com/NVIDIA-dev/cccl_private/issues/639
+// ============================================================================
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include <thrust/device_vector.h>
+
+#include <cuda/iterator>
+#include <cuda/random>
+#include <cuda/std/algorithm.max.h>
+#include <cuda/std/algorithm.min.h>
+#include <cuda/std/algorithm.transform.h>
+#include <cuda/std/cmath>
+#include <cuda/std/execution>
+#include <cuda/std/random>
+#include <cuda/std/type_traits>
+
+#include "../../common.h"
+#include "bfloat16.h"
+
+// ============================================================================
+// at::opmath_type<T> — the compute type for intermediate math
+// float for both float and BFloat16 (ATen/OpMathType.h)
+// ============================================================================
+
+template <typename T>
+struct opmath_type_impl
+{
+  using type = T;
+};
+template <>
+struct opmath_type_impl<BFloat16>
+{
+  using type = float;
+};
+template <typename T>
+using opmath_type = typename opmath_type_impl<T>::type;
+
+// ============================================================================
+// Replicate ATen/c10 helpers without ATen dependencies.
+// Each wrapper is annotated with the ATen source it replicates.
+// ============================================================================
+
+// c10::div_floor_floating (c10/util/generic_math.h:34)
+template <typename scalar_t>
+__device__ __forceinline__ scalar_t div_floor_floating(scalar_t a, scalar_t b)
+{
+  if (b == 0)
+  {
+    return a / b;
+  }
+
+  auto mod = std::fmod(a, b);
+  auto div = (a - mod) / b;
+  if ((mod != 0) && (b < 0) != (mod < 0))
+  {
+    div -= scalar_t(1);
+  }
+
+  scalar_t floordiv;
+  if (div != 0)
+  {
+    floordiv = std::floor(div);
+    if (div - floordiv > scalar_t(0.5))
+    {
+      floordiv += scalar_t(1.0);
+    }
+  }
+  else
+  {
+    floordiv = ::copysignf(scalar_t(0), a / b);
+  }
+  return floordiv;
+}
+
+// is_lerp_weight_small + lerp (native/Lerp.h:11,21)
+template <typename scalar_t>
+__device__ __forceinline__ bool is_lerp_weight_small(scalar_t weight)
+{
+  return std::abs(weight) < scalar_t(0.5);
+}
+
+template <typename scalar_t, typename weight_t>
+__device__ __forceinline__ scalar_t aten_lerp(scalar_t self_, scalar_t end_, weight_t weight_)
+{
+  using opmath_t        = opmath_type<scalar_t>;
+  using opmath_weight_t = opmath_type<weight_t>;
+
+  opmath_t self          = self_;
+  opmath_t end           = end_;
+  opmath_weight_t weight = weight_;
+
+  return is_lerp_weight_small(weight) ? self + weight * (end - self) : end - (end - self) * (opmath_t(1) - weight);
+}
+
+// pointwise_op_impl (native/cuda/DeviceAddCmulCdiv.cuh:9)
+template <typename opmath_t, typename Op>
+__device__ __forceinline__ opmath_t
+pointwise_op_impl(opmath_t input, opmath_t tensor1, opmath_t tensor2, opmath_t alpha, Op op)
+{
+  if (alpha == opmath_t(1))
+  {
+    if constexpr (std::is_same_v<Op, std::multiplies<opmath_t>> && std::is_floating_point_v<opmath_t>)
+    {
+      return std::fma(tensor1, tensor2, input);
+    }
+    else
+    {
+      return input + op(tensor1, tensor2);
+    }
+  }
+  if constexpr (std::is_floating_point_v<opmath_t>)
+  {
+    return std::fma(alpha, op(tensor1, tensor2), input);
+  }
+  else
+  {
+    return input + alpha * op(tensor1, tensor2);
+  }
+}
+
+// DivFunctor (native/cuda/BinaryInternal.h:20)
+template <typename scalar_t>
+struct DivFunctor
+{
+  __device__ scalar_t operator()(scalar_t a, scalar_t b) const
+  {
+    return a / b;
+  }
+};
+
+// MulFunctor (native/cuda/BinaryInternal.h:27)
+template <typename T>
+struct MulFunctor
+{
+  __device__ T operator()(T a, T b) const
+  {
+    return a * b;
+  }
+};
+
+// CUDAFunctorOnSelf_add — torchgen-generated ufunc functor for add(tensor, scalar)
+// (torchgen/dest/ufunc.py, native/ufunc/add.h:14)
+template <typename scalar_t>
+struct CUDAFunctorOnSelf_add
+{
+  using opmath_t = opmath_type<scalar_t>;
+  opmath_t other_;
+  opmath_t alpha_;
+  CUDAFunctorOnSelf_add(opmath_t other, opmath_t alpha)
+      : other_(other)
+      , alpha_(alpha)
+  {}
+  __device__ scalar_t operator()(scalar_t self) const
+  {
+    return static_cast<opmath_t>(self) + alpha_ * other_;
+  }
+};
+
+// CUDAFunctor_add — torchgen-generated ufunc functor for add(tensor, tensor)
+// (torchgen/dest/ufunc.py, native/ufunc/add.h:14)
+template <typename scalar_t>
+struct CUDAFunctor_add
+{
+  using opmath_t = opmath_type<scalar_t>;
+  opmath_t alpha_;
+  CUDAFunctor_add(opmath_t alpha)
+      : alpha_(alpha)
+  {}
+  __device__ scalar_t operator()(scalar_t self, scalar_t other) const
+  {
+    return static_cast<opmath_t>(self) + alpha_ * static_cast<opmath_t>(other);
+  }
+};
+
+// AbsFunctor (native/cuda/AbsKernel.cu:11)
+template <typename scalar_t>
+struct AbsFunctor
+{
+  __device__ __forceinline__ scalar_t operator()(const scalar_t a) const
+  {
+    return std::abs(a);
+  }
+};
+
+// CompareFunctor (native/cuda/CompareKernels.cu:14 / 17)
+enum class OpType
+{
+  GE,
+  GT,
+  LE,
+  LT
+};
+
+template <typename scalar_t>
+struct CompareFunctor
+{
+  constexpr CompareFunctor(OpType op)
+      : op_(op) {};
+  OpType op_;
+  __device__ __forceinline__ bool operator()(scalar_t a, scalar_t b) const
+  {
+    if (op_ == OpType::GE)
+    {
+      return a >= b;
+    }
+    else if (op_ == OpType::GT)
+    {
+      return a > b;
+    }
+    else if (op_ == OpType::LE)
+    {
+      return a <= b;
+    }
+    else
+    { // LT
+      return a < b;
+    }
+  }
+};
+
+// ============================================================================
+// RNG helpers
+// ============================================================================
+
+template <typename T>
+struct normal_gen
+{
+  float mean, stddev;
+  int64_t offset;
+  __host__ __device__ T operator()(int64_t idx) const
+  {
+    cuda::pcg64 rng(42);
+    rng.discard(offset + idx);
+    cuda::std::normal_distribution<float> dist(mean, stddev);
+    return T(dist(rng));
+  }
+};
+
+template <typename T>
+void fill_normal(thrust::device_vector<T>& v, int64_t n, int buf_idx)
+{
+  v.resize(n);
+  cuda::std::transform(
+    cuda::execution::gpu,
+    cuda::counting_iterator<int64_t, int64_t>(0),
+    cuda::counting_iterator<int64_t, int64_t>(n),
+    v.begin(),
+    normal_gen<T>{0.0f, 1.0f, buf_idx * n});
+}
+
+// ============================================================================
+// Helper to call DeviceTransform::Transform with the tuning policy
+// ============================================================================
+
+template <typename... Inputs, typename Output, typename TransformOp>
+void transform(cuda::std::tuple<Inputs...> inputs, Output output, int64_t n, TransformOp op, cudaStream_t stream)
+{
+  auto env = cuda::std::execution::env{
+    cuda::stream_ref{stream}
+#if !TUNE_BASE
+    ,
+    cuda::execution::tune(policy_selector{})
+#endif // !TUNE_BASE
+  };
+  cub::DeviceTransform::Transform(inputs, output, n, op, env);
+}
+
+template <typename Input, typename Output, typename TransformOp>
+void transform(Input input, Output output, int64_t n, TransformOp op, cudaStream_t stream)
+{
+  transform(cuda::std::make_tuple(input), output, n, op, stream);
+}
+
+// ============================================================================
+// Element types
+// ============================================================================
+
+#ifdef TUNE_T
+using element_types = nvbench::type_list<TUNE_T>;
+#else
+using element_types = nvbench::type_list<float, BFloat16>;
+#endif
+
+// ============================================================================
+// many_inputs_many_instructions
+//
+// div_floor -> div_trunc -> div -> atan2 -> hypot ->
+// xlogy -> xlog1py -> logaddexp -> logaddexp2 -> pow
+// 11 inputs, 10 binary ops
+// ============================================================================
+
+template <typename T>
+static void many_inputs_many_instructions(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+
+  constexpr int num_in = 11;
+  thrust::device_vector<T> in[num_in];
+  for (int i = 0; i < num_in; i++)
+  {
+    fill_normal(in[i], n, i);
+  }
+  thrust::device_vector<T> tmpA(n, thrust::no_init), tmpB(n, thrust::no_init);
+
+  T* d_in[num_in];
+  for (int i = 0; i < num_in; i++)
+  {
+    d_in[i] = thrust::raw_pointer_cast(in[i].data());
+  }
+  T* d_a = thrust::raw_pointer_cast(tmpA.data());
+  T* d_b = thrust::raw_pointer_cast(tmpB.data());
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(20L * n);
+  state.add_global_memory_writes<T>(10L * n);
+
+  // logaddexp2 captures inv_log_2 — native/cuda/LogAddExpKernel.cu:272
+  using opmath_t       = opmath_type<T>;
+  const auto inv_log_2 = static_cast<opmath_t>(1.0 / 0.693147180559945309417232121458176);
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](const nvbench::launch& launch) {
+    const auto s = launch.get_stream().get_stream();
+
+    // div_floor: native/cuda/BinaryDivFloorKernel.cu:72, helper c10/util/generic_math.h:34
+    transform(
+      cuda::std::make_tuple(d_in[0], d_in[1]),
+      d_a,
+      n,
+      [] __device__(T a, T b) -> T {
+        return div_floor_floating(a, b);
+      },
+      s);
+
+    // div_trunc: native/cuda/BinaryDivTruncKernel.cu:42
+    transform(
+      cuda::std::make_tuple(d_a, d_in[2]),
+      d_b,
+      n,
+      [] __device__(T a, T b) -> T {
+        return std::trunc(a / b);
+      },
+      s);
+
+    // div: native/cuda/BinaryDivTrueKernel.cu:54, DivFunctor in native/cuda/BinaryInternal.h:20
+    transform(cuda::std::make_tuple(d_b, d_in[3]), d_a, n, DivFunctor<T>(), s);
+
+    // atan2: native/cuda/BinaryGeometricKernels.cu:18
+    transform(
+      cuda::std::make_tuple(d_a, d_in[4]),
+      d_b,
+      n,
+      [] __device__(T a, T b) -> T {
+        return ::atan2(a, b);
+      },
+      s);
+
+    // hypot: native/cuda/BinaryGeometricKernels.cu:29
+    transform(
+      cuda::std::make_tuple(d_b, d_in[5]),
+      d_a,
+      n,
+      [] __device__(T a, T b) -> T {
+        return ::hypot(a, b);
+      },
+      s);
+
+    // xlogy: native/cuda/BinaryMiscOpsKernels.cu:46
+    transform(
+      cuda::std::make_tuple(d_a, d_in[6]),
+      d_b,
+      n,
+      [] __device__(T x, T y) -> T {
+        if (::isnan(static_cast<float>(y)))
+        {
+          return NAN;
+        }
+        if (x == 0)
+        {
+          return 0;
+        }
+        return x * std::log(y);
+      },
+      s);
+
+    // xlog1py: native/cuda/BinaryMiscOpsKernels.cu:60
+    transform(
+      cuda::std::make_tuple(d_b, d_in[7]),
+      d_a,
+      n,
+      [] __device__(T x, T y) -> T {
+        if (::isnan(static_cast<float>(y)))
+        {
+          return NAN;
+        }
+        if (x == 0)
+        {
+          return 0;
+        }
+        return x * std::log1p(y);
+      },
+      s);
+
+    // logaddexp: native/cuda/LogAddExpKernel.cu:253
+    transform(
+      cuda::std::make_tuple(d_a, d_in[8]),
+      d_b,
+      n,
+      [] __device__(T a_, T b_) -> T {
+        using opmath_t = opmath_type<T>;
+        const auto a   = static_cast<opmath_t>(a_);
+        const auto b   = static_cast<opmath_t>(b_);
+        if (::isinf(a) && a == b)
+        {
+          return a;
+        }
+        else
+        {
+          const auto m = ::max(a, b);
+          return m + ::log1p(::exp(-::abs(a - b)));
+        }
+      },
+      s);
+
+    // logaddexp2: native/cuda/LogAddExpKernel.cu:272
+    transform(
+      cuda::std::make_tuple(d_b, d_in[9]),
+      d_a,
+      n,
+      [inv_log_2] __device__(T a_, T b_) -> T {
+        using opmath_t = opmath_type<T>;
+        const auto a   = static_cast<opmath_t>(a_);
+        const auto b   = static_cast<opmath_t>(b_);
+        if (::isinf(a) && a == b)
+        {
+          return a;
+        }
+        else
+        {
+          const auto m = ::max(a, b);
+          return m + ::log1p(::exp2(-::abs(a - b))) * inv_log_2;
+        }
+      },
+      s);
+
+    // pow (tensor,tensor): native/cuda/PowKernel.cu:136, helper native/cuda/Pow.cuh:40
+    transform(
+      cuda::std::make_tuple(d_a, d_in[10]),
+      d_b,
+      n,
+      [] __device__(T base, T exp) -> T {
+        return cuda::std::pow(base, exp);
+      },
+      s);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+// ============================================================================
+// many_inputs_few_instructions
+//
+// mse_loss -> smooth_l1_loss -> huber_loss -> clamp_min ->
+// mul -> add -> addcmul -> lerp(scalar) -> lerp(tensor) -> greater
+// 13 inputs, 8 binary ops + 2 ternary ops
+// ============================================================================
+
+template <typename T>
+static void many_inputs_few_instructions(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+
+  constexpr int num_in = 13;
+  thrust::device_vector<T> in[num_in];
+  for (int i = 0; i < num_in; i++)
+  {
+    fill_normal(in[i], n, i);
+  }
+  thrust::device_vector<T> tmpA(n, thrust::no_init), tmpB(n, thrust::no_init);
+
+  T* d_in[num_in];
+  for (int i = 0; i < num_in; i++)
+  {
+    d_in[i] = thrust::raw_pointer_cast(in[i].data());
+  }
+  T* d_a = thrust::raw_pointer_cast(tmpA.data());
+  T* d_b = thrust::raw_pointer_cast(tmpB.data());
+
+  // 8 binary (16 reads) + 2 ternary (6 reads) = 22 reads, 10 writes
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(22L * n);
+  state.add_global_memory_writes<T>(10L * n);
+
+  // Captured scalar parameters, matching how ATen sets them up before gpu_kernel
+  using opmath_t = opmath_type<T>;
+  T beta_val(1.0); // smooth_l1: scalar_t beta_val(beta)
+  T delta_val(1.0); // huber: scalar_t delta_val(delta)
+  // note: opmath_type is same as at::acc_type<scalar_t, true> here
+  using accscalar_t     = opmath_type<T>; // addcmul: at::acc_type<scalar_t, true>
+  const auto alpha      = accscalar_t(1); // addcmul: value.to<accscalar_t>()
+  const auto weight_val = opmath_t(4.0); // lerp scalar: weight.to<opmath_t>()
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](const nvbench::launch& launch) {
+    const auto s = launch.get_stream().get_stream();
+
+    // mse_loss: native/cuda/BinaryMiscOpsKernels.cu:37
+    transform(
+      cuda::std::make_tuple(d_in[0], d_in[1]),
+      d_a,
+      n,
+      [] __device__(T a, T b) -> T {
+        auto diff = a - b;
+        return diff * diff;
+      },
+      s);
+
+    // smooth_l1_loss(beta=1.0): native/cuda/BinaryMiscOpsKernels.cu:19
+    transform(
+      cuda::std::make_tuple(d_a, d_in[2]),
+      d_b,
+      n,
+      [beta_val] __device__(T a, T b) -> T {
+        auto z = ::abs(a - b);
+        return z < beta_val ? T(0.5) * z * z / beta_val : z - T(0.5) * beta_val;
+      },
+      s);
+
+    // huber_loss(delta=1.0): native/cuda/BinaryMiscOpsKernels.cu:29
+    transform(
+      cuda::std::make_tuple(d_b, d_in[3]),
+      d_a,
+      n,
+      [delta_val] __device__(T a, T b) -> T {
+        auto z = ::abs(a - b);
+        return z < delta_val ? T(0.5) * z * z : delta_val * (z - T(0.5) * delta_val);
+      },
+      s);
+
+    // clamp(min=tensor) -> maximum: native/cuda/MaxMinElementwiseKernel.cu:28
+    transform(
+      cuda::std::make_tuple(d_a, d_in[4]),
+      d_b,
+      n,
+      [] __device__(T a, T b) -> T {
+        if (a != a)
+        {
+          return a;
+        }
+        else if (b != b)
+        {
+          return b;
+        }
+        else
+        {
+          return ::max(a, b);
+        }
+      },
+      s);
+
+    // mul: native/cuda/BinaryMulKernel.cu:39, MulFunctor in native/cuda/BinaryInternal.h:27
+    using mul_opmath_t = opmath_type<T>;
+    transform(cuda::std::make_tuple(d_b, d_in[5]), d_a, n, MulFunctor<mul_opmath_t>(), s);
+
+    // add(alpha=1): native/ufunc/add.h:14, torchgen/dest/ufunc.py
+    transform(cuda::std::make_tuple(d_a, d_in[6]), d_b, n, CUDAFunctor_add<T>(1.0), s);
+
+    // addcmul(value=1): native/cuda/PointwiseOpsKernel.cu:87, native/cuda/DeviceAddCmulCdiv.cuh:9
+    transform(
+      cuda::std::make_tuple(d_b, d_in[7], d_in[8]),
+      d_a,
+      n,
+      [alpha] __device__(T a, T b, T c) -> T {
+        return pointwise_op_impl<accscalar_t>(a, b, c, alpha, cuda::std::multiplies<accscalar_t>());
+      },
+      s);
+
+    // lerp(weight=4.0): native/cuda/Lerp.cu:130, native/Lerp.h:21
+    transform(
+      cuda::std::make_tuple(d_a, d_in[9]),
+      d_b,
+      n,
+      [=] __device__(T self_val, T end_val) {
+        return aten_lerp(self_val, end_val, weight_val);
+      },
+      s);
+
+    // lerp(weight=tensor): native/cuda/Lerp.cu:76, native/Lerp.h:21
+    transform(
+      cuda::std::make_tuple(d_b, d_in[10], d_in[11]),
+      d_a,
+      n,
+      [] __device__(T self_val, T end_val, T weight_val) -> T {
+        return aten_lerp(self_val, end_val, weight_val);
+      },
+      s);
+
+    // note: even though output is bool, we use d_b as output because
+    // it must hold at least enough memory per element for bool (1 byte)
+    // greater: native/cuda/CompareKernels.cu:69
+    CompareFunctor<T> comp_f(OpType::GT);
+    transform(cuda::std::make_tuple(d_a, d_in[12]), d_b, n, comp_f, s);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+// ============================================================================
+// few_inputs_many_instructions
+//
+// pow(2.5) -> tanh -> sin -> cos -> softplus ->
+// silu -> mish -> elu -> gelu -> logsigmoid
+// 1 input, 10 unary ops
+// ============================================================================
+
+template <typename T>
+static void few_inputs_many_instructions(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+
+  thrust::device_vector<T> input(n, thrust::no_init);
+  fill_normal(input, n, 0);
+  thrust::device_vector<T> tmpA(n, thrust::no_init), tmpB(n, thrust::no_init);
+
+  T* d_in = thrust::raw_pointer_cast(input.data());
+  T* d_a  = thrust::raw_pointer_cast(tmpA.data());
+  T* d_b  = thrust::raw_pointer_cast(tmpB.data());
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(10L * n);
+  state.add_global_memory_writes<T>(10L * n);
+
+  // Captured scalar parameters
+  using opmath_t        = opmath_type<T>;
+  const auto exp_val    = T(2.5); // pow: exp_scalar.to<scalar_t>()
+  const auto beta       = opmath_t(1); // softplus: beta_.to<opmath_t>()
+  const auto threshold  = opmath_t(20); // softplus: threshold_.to<opmath_t>()
+  const auto negcoef    = opmath_t(1) * opmath_t(1); // elu: alpha * scale
+  const auto poscoef    = opmath_t(1); // elu: scale
+  const auto negiptcoef = opmath_t(1); // elu: input_scale
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](const nvbench::launch& launch) {
+    const auto s = launch.get_stream().get_stream();
+
+    // pow(scalar=2.5): native/cuda/PowKernel.cu:163, helper native/cuda/Pow.cuh:40
+    transform(
+      d_in,
+      d_a,
+      n,
+      [=] __device__(T base) -> T {
+        return cuda::std::pow(base, exp_val);
+      },
+      s);
+
+    // tanh: native/cuda/UnaryGeometricTanhKernel.cu:50
+    transform(
+      d_a,
+      d_b,
+      n,
+      [] __device__(T a) -> T {
+        return ::tanh(a);
+      },
+      s);
+
+    // sin: native/cuda/UnaryGeometricSinKernel.cu:50
+    transform(
+      d_b,
+      d_a,
+      n,
+      [] __device__(T a) -> T {
+        return ::sin(a);
+      },
+      s);
+
+    // cos: native/cuda/UnaryGeometricCosKernel.cu:50
+    transform(
+      d_a,
+      d_b,
+      n,
+      [] __device__(T a) -> T {
+        return ::cos(a);
+      },
+      s);
+
+    // softplus(beta=1, threshold=20): native/cuda/ActivationSoftplusKernel.cu:35
+    transform(
+      d_b,
+      d_a,
+      n,
+      [beta, threshold] __device__(T a) -> T {
+        using opmath_t = opmath_type<T>;
+        opmath_t aop   = static_cast<opmath_t>(a);
+        return (aop * beta) > threshold ? aop : (::log1p(std::exp(aop * beta))) / beta;
+      },
+      s);
+
+    // silu: native/cuda/ActivationSiluKernel.cu:30
+    transform(
+      d_a,
+      d_b,
+      n,
+      [] __device__(T x) -> T {
+        using opmath_t       = opmath_type<T>;
+        const opmath_t x_acc = static_cast<opmath_t>(x);
+        return x_acc / (opmath_t(1) + ::exp(-x_acc));
+      },
+      s);
+
+    // mish: native/cuda/ActivationMishKernel.cu:29
+    transform(
+      d_b,
+      d_a,
+      n,
+      [] __device__(T x) -> T {
+        using opmath_t       = opmath_type<T>;
+        const opmath_t x_acc = static_cast<opmath_t>(x);
+        return x_acc * ::tanhf(::log1pf(::expf(x_acc)));
+      },
+      s);
+
+    // elu(alpha=1, scale=1, input_scale=1): native/cuda/ActivationEluKernel.cu:37
+    transform(
+      d_a,
+      d_b,
+      n,
+      [negcoef, poscoef, negiptcoef] __device__(T a) -> T {
+        using opmath_t = opmath_type<T>;
+        opmath_t aop   = static_cast<opmath_t>(a);
+        return aop > 0 ? aop * poscoef : std::expm1(aop * negiptcoef) * negcoef;
+      },
+      s);
+
+    // gelu(approximate='none'): native/cuda/ActivationGeluKernel.cu:35
+    transform(
+      d_b,
+      d_a,
+      n,
+      [] __device__(T x) -> T {
+        using opmath_t            = opmath_type<T>;
+        constexpr opmath_t kAlpha = M_SQRT1_2;
+        return static_cast<opmath_t>(x) * opmath_t(0.5) * (opmath_t(1) + ::erf(static_cast<opmath_t>(x) * kAlpha));
+      },
+      s);
+
+    // logsigmoid: native/cuda/ActivationLogSigmoidKernel.cu:30
+    transform(
+      d_a,
+      d_b,
+      n,
+      [] __device__(T in_) -> T {
+        using opmath_t    = opmath_type<T>;
+        const opmath_t in = in_;
+        const auto min    = cuda::std::min(opmath_t(0), in);
+        const auto z      = std::exp(-std::abs(in));
+        return min - std::log1p(z);
+      },
+      s);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+// ============================================================================
+// few_inputs_few_instructions
+//
+// add(0.5) -> neg -> clamp(-2,1) -> abs -> mul(1.5) ->
+// leaky_relu -> hardswish -> hardshrink -> hardsigmoid -> gt(0)
+// 1 input, 10 unary ops
+// ============================================================================
+
+template <typename T>
+static void few_inputs_few_instructions(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+
+  thrust::device_vector<T> input(n, thrust::no_init);
+  fill_normal(input, n, 0);
+  thrust::device_vector<T> tmpA(n, thrust::no_init), tmpB(n, thrust::no_init);
+
+  T* d_in = thrust::raw_pointer_cast(input.data());
+  T* d_a  = thrust::raw_pointer_cast(tmpA.data());
+  T* d_b  = thrust::raw_pointer_cast(tmpB.data());
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(10L * n);
+  state.add_global_memory_writes<T>(10L * n);
+
+  // Captured scalar parameters
+  using opmath_t = opmath_type<T>;
+
+  // clamp: native/cuda/TensorCompare.cu:58
+  const auto lim0_val = opmath_t(-2);
+  const auto lim1_val = opmath_t(1);
+  const auto minmax   = 2; // 0=Min, 1=Max, 2=MinMax
+
+  // mul scalar: MulFunctor via BUnaryFunctor with captured scalar
+  const auto mul_scalar = opmath_t(1.5);
+
+  // leaky_relu: native/cuda/ActivationLeakyReluKernel.cu:31
+  const auto negval = opmath_t(0.01); // negval_.to<opmath_t>()
+
+  // hardswish: native/cuda/ActivationHardswishKernel.cu:25
+  const opmath_t zero(0.0f);
+  const opmath_t one_sixth(1.0f / 6.0f);
+  const opmath_t three(3.0f);
+  const opmath_t six(6.0f);
+
+  // hardshrink: native/cuda/ActivationHardshrinkKernel.cu:29
+  const auto lambd = T(0.5); // value.to<scalar_t>()
+
+  // gt scalar: native/cuda/CompareKernels.cu:47
+  const T rhs(0);
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](const nvbench::launch& launch) {
+    const auto s = launch.get_stream().get_stream();
+
+    // add(scalar, alpha=1): native/ufunc/add.h:14
+    transform(d_in, d_a, n, CUDAFunctorOnSelf_add<T>(T(0.5), T(1)), s);
+
+    // neg: native/cuda/UnarySignKernels.cu:54
+    transform(
+      d_a,
+      d_b,
+      n,
+      [] __device__(T a) -> T {
+        return -a;
+      },
+      s);
+
+    // clamp(min=-2, max=1): native/cuda/TensorCompare.cu:58 (MinMax branch)
+    transform(
+      d_b,
+      d_a,
+      n,
+      [=] __device__(T v) -> T {
+        using opmath_t = opmath_type<T>;
+        if (::isnan(static_cast<opmath_t>(v)))
+        {
+          return v;
+        }
+        else if (minmax == 0)
+        {
+          return ::max(static_cast<opmath_t>(v), lim0_val);
+        }
+        else if (minmax == 1)
+        {
+          return ::min(static_cast<opmath_t>(v), lim0_val);
+        }
+        else
+        {
+          return ::min(::max(static_cast<opmath_t>(v), lim0_val), lim1_val);
+        }
+      },
+      s);
+
+    // abs: native/cuda/AbsKernel.cu:39, AbsFunctor:11
+    transform(d_a, d_b, n, AbsFunctor<T>(), s);
+
+    // mul(scalar=1.5): native/cuda/BinaryMulKernel.cu:39, MulFunctor via BUnaryFunctor
+    transform(
+      d_b,
+      d_a,
+      n,
+      [mul_scalar] __device__(T a) -> T {
+        return MulFunctor<opmath_t>()(a, mul_scalar);
+      },
+      s);
+
+    // leaky_relu(slope=0.01): native/cuda/ActivationLeakyReluKernel.cu:31
+    transform(
+      d_a,
+      d_b,
+      n,
+      [negval] __device__(T a) -> T {
+        using opmath_t = opmath_type<T>;
+        opmath_t aop   = static_cast<opmath_t>(a);
+        return aop > opmath_t(0) ? aop : aop * negval;
+      },
+      s);
+
+    // hardswish: native/cuda/ActivationHardswishKernel.cu:25
+    transform(
+      d_b,
+      d_a,
+      n,
+      [zero, one_sixth, three, six] __device__(T self_val) -> T {
+        using opmath_t = opmath_type<T>;
+        opmath_t x     = static_cast<opmath_t>(self_val);
+        return x * cuda::std::min(cuda::std::max(x + three, zero), six) * one_sixth;
+      },
+      s);
+
+    // hardshrink(lambd=0.5): native/cuda/ActivationHardshrinkKernel.cu:29
+    transform(
+      d_a,
+      d_b,
+      n,
+      [lambd] __device__(T a) -> T {
+        return (a >= -lambd && a <= lambd) ? T(0) : a;
+      },
+      s);
+
+    // hardsigmoid: native/cuda/ActivationHardsigmoidKernel.cu:30
+    transform(
+      d_b,
+      d_a,
+      n,
+      [zero, one_sixth, three, six] __device__(T self_val) -> T {
+        using opmath_t = opmath_type<T>;
+        opmath_t x     = static_cast<opmath_t>(self_val);
+        return cuda::std::min<opmath_t>(cuda::std::max<opmath_t>(x + three, zero), six) * one_sixth;
+      },
+      s);
+
+    // gt(scalar=0): native/cuda/CompareKernels.cu:47
+    CompareFunctor<T> comp_f(OpType::GT);
+    transform(
+      d_a,
+      d_b,
+      n,
+      [=] __device__(T lhs) -> T {
+        return comp_f(lhs, rhs);
+      },
+      s);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(many_inputs_many_instructions, NVBENCH_TYPE_AXES(element_types))
+  .set_name("many_inputs_many_instructions")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));
+
+NVBENCH_BENCH_TYPES(many_inputs_few_instructions, NVBENCH_TYPE_AXES(element_types))
+  .set_name("many_inputs_few_instructions")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));
+
+NVBENCH_BENCH_TYPES(few_inputs_many_instructions, NVBENCH_TYPE_AXES(element_types))
+  .set_name("few_inputs_many_instructions")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));
+
+NVBENCH_BENCH_TYPES(few_inputs_few_instructions, NVBENCH_TYPE_AXES(element_types))
+  .set_name("few_inputs_few_instructions")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/transform/babelstream.cu

@@ -0,0 +1,191 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+#ifdef TUNE_T
+using element_types = nvbench::type_list<TUNE_T>;
+#else
+using element_types =
+  nvbench::type_list<std::int8_t,
+                     std::int16_t,
+                     float,
+                     double
+#  if _CCCL_HAS_INT128()
+                     ,
+                     __int128
+#  endif
+                     >;
+#endif
+
+// BabelStream uses 2^25, H200 can fit 2^31 int128s
+// 2^20 chars / 2^16 int128 saturate V100 (min_bytes_in_flight =12 * SM count =80)
+// 2^21 chars / 2^17 int128 saturate A100 (min_bytes_in_flight =16 * SM count =108)
+// 2^23 chars / 2^19 int128 saturate H100/H200 HBM3 (min_bytes_in_flight =32or48 * SM count =132)
+// inline auto array_size_powers = std::vector<nvbench::int64_t>{28};
+inline auto array_size_powers = nvbench::range(16, 32, 4);
+
+// Modified from BabelStream to also work for integers and to make nstream maintain a consistent workload since it
+// overwrites one input array. If the data changed at each iteration, the performance would be unstable.
+inline constexpr auto startA      = 11; // BabelStream: 0.1
+inline constexpr auto startB      = 2; // BabelStream: 0.2
+inline constexpr auto startC      = 1; // BabelStream: 0.1
+inline constexpr auto startScalar = -2; // BabelStream: 0.4
+
+static_assert(startA == (startA + startB + startScalar * startC), "nstream must have a consistent workload");
+
+template <typename T>
+static void mul(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(n);
+
+  const T scalar = startScalar;
+  bench_transform(
+    state, cuda::std::tuple{c.begin() + unaligned}, b.begin() + unaligned, n, [=] _CCCL_DEVICE(const T& ci) {
+      return ci * scalar;
+    });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(mul, NVBENCH_TYPE_AXES(element_types))
+  .set_name("mul")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);
+
+template <typename T>
+static void add(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+
+  thrust::device_vector<T> a(n + unaligned, startA);
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(
+    state,
+    cuda::std::tuple{a.begin() + unaligned, b.begin() + unaligned},
+    c.begin() + unaligned,
+    n,
+    [] _CCCL_DEVICE(const T& ai, const T& bi) -> T {
+      return ai + bi;
+    });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(add, NVBENCH_TYPE_AXES(element_types))
+  .set_name("add")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);
+
+template <typename T>
+static void triad(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+
+  thrust::device_vector<T> a(n + unaligned, startA);
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+  const T scalar = startScalar;
+  bench_transform(
+    state,
+    cuda::std::tuple{b.begin() + unaligned, c.begin() + unaligned},
+    a.begin() + unaligned,
+    n,
+    [=] _CCCL_DEVICE(const T& bi, const T& ci) {
+      return bi + scalar * ci;
+    });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(triad, NVBENCH_TYPE_AXES(element_types))
+  .set_name("triad")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);
+
+template <typename T>
+static void nstream(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+
+  thrust::device_vector<T> a(n + unaligned, startA);
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(3 * n);
+  state.add_global_memory_writes<T>(n);
+  const T scalar = startScalar;
+  bench_transform(
+    state,
+    cuda::std::tuple{a.begin() + unaligned, b.begin() + unaligned, c.begin() + unaligned},
+    a.begin() + unaligned,
+    n,
+    [=] _CCCL_DEVICE(const T& ai, const T& bi, const T& ci) {
+      return ai + bi + scalar * ci;
+    });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(nstream, NVBENCH_TYPE_AXES(element_types))
+  .set_name("nstream")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);

cccl_upstream/cub/benchmarks/bench/transform/common.h

@@ -0,0 +1,96 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+// keep checks at the top so compilation of discarded variants fails really fast
+#include <cub/device/dispatch/dispatch_transform.cuh>
+#if !TUNE_BASE
+#  if _CCCL_PP_COUNT(__CUDA_ARCH_LIST__) != 1
+#    error "When tuning, this benchmark does not support being compiled for multiple architectures"
+#  endif
+#  if TUNE_ALGORITHM == 3
+#    if (__CUDA_ARCH_LIST__) < 900
+#      error "Cannot compile algorithm 3 (ublkcp) below sm90"
+#    endif
+#  endif // TUNE_ALGORITHM == 3
+#endif // !TUNE_BASE
+
+#include <cub/util_namespace.cuh>
+
+#include <cuda/__numeric/narrow.h>
+#include <cuda/std/cstdint>
+#include <cuda/std/type_traits>
+
+#include <stdexcept>
+
+#include <nvbench_helper.cuh>
+
+#if !TUNE_BASE
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability cc) const -> cub::TransformPolicy
+  {
+    const int min_bytes_in_flight = cub::detail::transform::cc_to_min_bytes_in_flight(cc) + TUNE_BIF_BIAS;
+#  if TUNE_ALGORITHM == 0 || TUNE_ALGORITHM == 1
+    // setup prefetch, since it's either used directly or the fallback to vectorized
+    auto algorithm                = cub::TransformAlgorithm::prefetch;
+    auto pref_policy              = cub::TransformPrefetchPolicy{};
+    pref_policy.threads_per_block = TUNE_THREADS;
+    pref_policy.unroll_factor     = TUNE_UNROLL_FACTOR;
+#    ifdef TUNE_PREFETCH_MULT
+    pref_policy.prefetch_byte_stride = 32 * TUNE_PREFETCH_MULT;
+#    endif //  TUNE_PREFETCH_MULT
+#    ifdef TUNE_ITEMS_PER_THREAD_NO_INPUT
+    pref_policy.items_per_thread_no_input = TUNE_ITEMS_PER_THREAD_NO_INPUT;
+#    endif // TUNE_ITEMS_PER_THREAD_NO_INPUT
+
+    // setup vectorized if requested
+    auto vec_policy = cub::TransformVectorizedPolicy{};
+#    if TUNE_ALGORITHM == 1
+    algorithm                    = cub::TransformAlgorithm::vectorized;
+    vec_policy.threads_per_block = TUNE_THREADS;
+    vec_policy.vec_size          = (1 << TUNE_VEC_SIZE_POW2);
+    vec_policy.items_per_thread  = vec_policy.vec_size * TUNE_UNROLL_FACTOR;
+#    endif
+    return {min_bytes_in_flight, algorithm, pref_policy, vec_policy, {}};
+#  elif TUNE_ALGORITHM == 2
+    constexpr auto algorithm = cub::TransformAlgorithm::ldgsts;
+    auto policy              = cub::TransformAsyncCopyPolicy{};
+    policy.threads_per_block = TUNE_THREADS;
+    policy.unroll_factor     = TUNE_UNROLL_FACTOR;
+    return {min_bytes_in_flight, algorithm, {}, {}, policy};
+#  elif TUNE_ALGORITHM == 3
+    constexpr auto algorithm = cub::TransformAlgorithm::ublkcp;
+    auto policy              = cub::TransformAsyncCopyPolicy{};
+    policy.threads_per_block = TUNE_THREADS;
+    policy.unroll_factor     = TUNE_UNROLL_FACTOR;
+    return {min_bytes_in_flight, algorithm, {}, {}, policy};
+#  else // TUNE_ALGORITHM
+#    error Policy hub does not yet implement the specified value for algorithm
+#  endif // TUNE_ALGORITHM
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename... RandomAccessIteratorsIn, typename RandomAccessIteratorOut, typename TransformOp>
+void bench_transform(nvbench::state& state,
+                     cuda::std::tuple<RandomAccessIteratorsIn...> inputs,
+                     RandomAccessIteratorOut output,
+                     ::cuda::std::int64_t num_items,
+                     TransformOp transform_op)
+{
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](const nvbench::launch& launch) {
+    cub::DeviceTransform::Transform(
+      inputs,
+      output,
+      num_items,
+      transform_op,
+      cuda::std::execution::env{::cuda::stream_ref{launch.get_stream().get_stream()}
+#if !TUNE_BASE
+                                ,
+                                cuda::execution::tune(policy_selector{})
+#endif // !TUNE_BASE
+      });
+  });
+}

cccl_upstream/cub/benchmarks/bench/transform/complex_cmp.cu

@@ -0,0 +1,51 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+// This benchmark tests overlapping memory regions for reading and is compute intensive
+
+static void compare_complex(nvbench::state& state)
+try
+{
+  const auto n                        = state.get_int64("Elements{io}");
+  thrust::device_vector<complex32> in = generate(n);
+  thrust::device_vector<bool> out(n - 1);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<complex32>(n);
+  state.add_global_memory_writes<bool>(n);
+
+  // the complex comparison needs lots of compute and transform reads from overlapping input
+  using compare_op = less_t;
+  bench_transform(state, cuda::std::tuple{in.begin(), in.begin() + 1}, out.begin(), n - 1, compare_op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH(compare_complex)
+  .set_name("compare_complex")
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/fib.cu

@@ -0,0 +1,78 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+// This benchmark is compute intensive with diverging threads
+
+template <class IndexT, class OutputT>
+struct fib_t
+{
+  __device__ OutputT operator()(IndexT n)
+  {
+    OutputT t1 = 0;
+    OutputT t2 = 1;
+
+    if (n < 1)
+    {
+      return t1;
+    }
+    if (n == 1)
+    {
+      return t1;
+    }
+    if (n == 2)
+    {
+      return t2;
+    }
+    for (IndexT i = 3; i <= n; ++i)
+    {
+      const auto next = t1 + t2;
+      t1              = t2;
+      t2              = next;
+    }
+    return t2;
+  }
+};
+static void fibonacci(nvbench::state& state)
+try
+{
+  using index_t                     = int64_t;
+  using output_t                    = uint32_t;
+  const auto n                      = state.get_int64("Elements{io}");
+  thrust::device_vector<index_t> in = generate(n, bit_entropy::_1_000, index_t{0}, index_t{42});
+  thrust::device_vector<output_t> out(n);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<index_t>(n);
+  state.add_global_memory_writes<output_t>(n);
+
+  bench_transform(state, cuda::std::tuple{in.begin()}, out.begin(), n, fib_t<index_t, output_t>{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH(fibonacci).set_name("fibonacci").add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/fill.cu

@@ -0,0 +1,64 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// for filling, we can only use the prefetch and the vectorized algorithm
+// %RANGE% TUNE_ALGORITHM alg 0:2:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_ITEMS_PER_THREAD_NO_INPUT ipt 1:32:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_ITEMS_PER_THREAD_NO_INPUT != 1)
+#  error "Non-prefetch algorithms require the no input items per thread to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1 || TUNE_VECTORS_PER_THREAD != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+template <typename T>
+struct return_constant
+{
+  T value;
+
+  _CCCL_DEVICE auto operator()() const -> T
+  {
+    return value;
+  }
+};
+
+template <typename T>
+static void fill(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  // A 32-bit offset type or the value 0 or 0xFF... have <1% performance impact
+  const auto value     = T{42};
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+  thrust::device_vector<T> out(n + unaligned);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(0);
+  state.add_global_memory_writes<T>(n);
+
+  bench_transform(state, cuda::std::tuple{}, out.begin() + unaligned, n, return_constant<T>{value});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(fill, NVBENCH_TYPE_AXES(integral_types))
+  .set_name("fill")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/grayscale.cu

@@ -0,0 +1,96 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+template <typename T>
+struct rgb_t
+{
+  T r;
+  T g;
+  T b;
+
+  __device__ T grayscale() const
+  {
+    static constexpr T w_r(0.2989);
+    static constexpr T w_g(0.587);
+    static constexpr T w_b(0.114);
+
+    return w_r * r + w_g * g + w_b * b;
+  }
+};
+
+template <typename T>
+struct transform_op_t
+{
+  __device__ T operator()(rgb_t<T> pixel) const
+  {
+    return pixel.grayscale();
+  }
+};
+
+template <typename T>
+static void grayscale(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  using pixel_t = rgb_t<T>;
+  const auto n  = state.get_int64("Elements{io}");
+
+  // Generate random RGB data by creating separate R, G, B vectors and combining them
+  thrust::device_vector<T> r_data = generate(n);
+  thrust::device_vector<T> g_data = generate(n);
+  thrust::device_vector<T> b_data = generate(n);
+
+  thrust::device_vector<pixel_t> input(n, thrust::no_init);
+  thrust::transform(
+    thrust::make_zip_iterator(r_data.begin(), g_data.begin(), b_data.begin()),
+    thrust::make_zip_iterator(r_data.end(), g_data.end(), b_data.end()),
+    input.begin(),
+    thrust::make_zip_function([] __device__(T r, T g, T b) {
+      return pixel_t{r, g, b};
+    }));
+
+  thrust::device_vector<T> output(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<pixel_t>(n);
+  state.add_global_memory_writes<T>(n);
+
+  bench_transform(state, cuda::std::tuple{input.begin()}, output.begin(), n, transform_op_t<T>{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+#ifdef TUNE_T
+using value_types = nvbench::type_list<TUNE_T>;
+#else
+using value_types = nvbench::type_list<float, double>;
+#endif
+
+NVBENCH_BENCH_TYPES(grayscale, NVBENCH_TYPE_AXES(value_types))
+  .set_name("grayscale")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/heavy.cu

@@ -0,0 +1,93 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+// This benchmark uses a LOT of registers and is compute intensive.
+
+template <int N>
+struct heavy_functor
+{
+  // we need to use an unsigned type so overflow in arithmetic wraps around
+  __device__ std::uint32_t operator()(std::uint32_t data) const
+  {
+    std::uint32_t reg[N];
+    reg[0] = data;
+    for (int i = 1; i < N; ++i)
+    {
+      reg[i] = reg[i - 1] * reg[i - 1] + 1;
+    }
+    for (int i = 0; i < N; ++i)
+    {
+      reg[i] = (reg[i] * reg[i]) % 19;
+    }
+    for (int i = 0; i < N; ++i)
+    {
+      reg[i] = reg[N - i - 1] * reg[i];
+    }
+    std::uint32_t x = 0;
+    for (int i = 0; i < N; ++i)
+    {
+      x += reg[i];
+    }
+    return x;
+  }
+};
+
+template <typename Heaviness>
+static void heavy(nvbench::state& state, nvbench::type_list<Heaviness>)
+try
+{
+  using value_t = std::uint32_t;
+  const auto n  = state.get_int64("Elements{io}");
+
+  thrust::device_vector<value_t> in = generate(n);
+  thrust::device_vector<value_t> out(n);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<value_t>(n);
+  state.add_global_memory_writes<value_t>(n);
+
+  bench_transform(state, cuda::std::tuple{in.begin()}, out.begin(), n, heavy_functor<Heaviness::value>{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+using ::cuda::std::integral_constant;
+#ifdef TUNE_Heaviness
+using heaviness = nvbench::type_list<TUNE_Heaviness>; // expands to "integral_constant<int, ...>"
+#else
+using heaviness =
+  nvbench::type_list<integral_constant<int, 32>,
+                     integral_constant<int, 64>,
+                     integral_constant<int, 128>,
+                     integral_constant<int, 256>>;
+#endif
+
+NVBENCH_BENCH_TYPES(heavy, NVBENCH_TYPE_AXES(heaviness))
+  .set_name("heavy")
+  .set_type_axes_names({"Heaviness{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/pytorch.cu

@@ -0,0 +1,209 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+#ifdef TUNE_T
+using element_types = nvbench::type_list<TUNE_T>;
+#else
+using element_types = nvbench::type_list<
+#  if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
+  __half,
+#  endif
+#  if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
+  __nv_bfloat16,
+#  endif
+  float>;
+#endif
+
+template <typename Op, typename T>
+static void unary(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+  thrust::device_vector<T> in(n, 1337);
+  thrust::device_vector<T> out(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(n);
+
+  bench_transform(state, cuda::std::tuple{in.begin()}, out.begin(), n, Op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+#define BENCHMARK_UNARY(func)                                               \
+  template <typename T>                                                     \
+  static void func##_bench(nvbench::state& state, nvbench::type_list<T> tl) \
+  {                                                                         \
+    unary<func##_op>(state, tl);                                            \
+  }                                                                         \
+                                                                            \
+  NVBENCH_BENCH_TYPES(func##_bench, NVBENCH_TYPE_AXES(element_types))       \
+    .set_name(#func)                                                        \
+    .set_type_axes_names({"T{ct}"})                                         \
+    .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));
+
+// See: https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/OpMathType.h
+using opmath_t = float;
+
+// See for example:
+// https://github.com/pytorch/pytorch/blob/5a48148c1ab83c1e3779283d904ba5744bbe8eb3/aten/src/ATen/native/cuda/ActivationLeakyReluKernel.cu#L28-L35
+struct relu_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T value) const
+  {
+    return static_cast<T>(static_cast<opmath_t>(value) > opmath_t{0} ? static_cast<opmath_t>(value) : opmath_t{0});
+  }
+};
+BENCHMARK_UNARY(relu);
+
+// See for example:
+// https://github.com/pytorch/pytorch/blob/5a48148c1ab83c1e3779283d904ba5744bbe8eb3/aten/src/ATen/native/cuda/UnarySpecialOpsKernel.cu#L152-L157
+struct sigmoid_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T value) const
+  {
+    return static_cast<T>(opmath_t{1} / (opmath_t{1} + ::cuda::std::exp(-static_cast<opmath_t>(value))));
+  }
+};
+BENCHMARK_UNARY(sigmoid);
+
+struct tanh_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T value) const
+  {
+    return ::cuda::std::tanh(value);
+  }
+};
+BENCHMARK_UNARY(tanh);
+
+// See for example:
+// https://github.com/pytorch/pytorch/blob/5a48148c1ab83c1e3779283d904ba5744bbe8eb3/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L21
+struct gelu_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T value) const
+  {
+    return static_cast<opmath_t>(value) * opmath_t{0.5}
+         * (opmath_t{1} + ::cuda::std::erf(static_cast<opmath_t>(value) * opmath_t{M_SQRT1_2}));
+  }
+};
+BENCHMARK_UNARY(gelu);
+
+struct sin_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T value) const
+  {
+    return ::cuda::std::sin(value);
+  }
+};
+BENCHMARK_UNARY(sin);
+
+struct exp_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T value) const
+  {
+    return ::cuda::std::exp(value);
+  }
+};
+BENCHMARK_UNARY(exp);
+
+template <typename Op, typename T>
+static void binary(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+  thrust::device_vector<T> in1(n, 1337);
+  thrust::device_vector<T> in2(n, 42);
+  thrust::device_vector<T> out(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+
+  bench_transform(state, cuda::std::tuple{in1.begin(), in2.begin()}, out.begin(), n, Op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+#define BENCHMARK_BINARY(func)                                              \
+  template <typename T>                                                     \
+  static void func##_bench(nvbench::state& state, nvbench::type_list<T> tl) \
+  {                                                                         \
+    binary<func##_op>(state, tl);                                           \
+  }                                                                         \
+                                                                            \
+  NVBENCH_BENCH_TYPES(func##_bench, NVBENCH_TYPE_AXES(element_types))       \
+    .set_name(#func)                                                        \
+    .set_type_axes_names({"T{ct}"})                                         \
+    .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));
+
+using add_op = cuda::std::plus<>;
+BENCHMARK_BINARY(add);
+
+using sub_op = cuda::std::minus<>;
+BENCHMARK_BINARY(sub);
+
+using mul_op = cuda::std::multiplies<>;
+BENCHMARK_BINARY(mul);
+
+using div_op = cuda::std::divides<>;
+BENCHMARK_BINARY(div);
+
+using le_op = cuda::std::less_equal<>;
+BENCHMARK_BINARY(le);
+
+using ge_op = cuda::std::greater_equal<>;
+BENCHMARK_BINARY(ge);
+
+struct fmin_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T a, T b) const
+  {
+    return ::cuda::std::fmin(a, b);
+  }
+};
+BENCHMARK_BINARY(fmin);
+
+struct fmax_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T a, T b) const
+  {
+    return ::cuda::std::fmax(a, b);
+  }
+};
+BENCHMARK_BINARY(fmax);

cccl_upstream/cub/benchmarks/bench/transform/tile/babelstream.cu

@@ -0,0 +1,216 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Tile variant of the BabelStream transform bench. The lambdas of the base benchmark are replaced by
+// named, stateless ops that register a tile_operator substitute (gated). Under --enable-tile +
+// CCCL_ENABLE_EXPERIMENTAL_TILE_TRANSFORM_DISPATCH the dispatch hook routes them to the tile kernel; otherwise this
+// is the standard CUB transform path. This file disappears once tile dispatch is fully transparent.
+
+#include "../common.h"
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+#  include <cuda_tile.h>
+#endif
+
+// Stateless scalar ops, used at the call site in both build modes. Constants are baked in so the ops
+// stay stateless (the tile substitute must be trivially default constructible): with startScalar == -2,
+// `c * scalar` is `-(c + c)`, `b + scalar * c` is `b - c - c`, etc.
+struct mul_op
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class B>
+  _CCCL_API auto operator()(B b) const
+  {
+    return -(b + b);
+  }
+};
+struct add_op
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class A, class B>
+  _CCCL_API auto operator()(A a, B b) const
+  {
+    return a + b;
+  }
+};
+struct triad_op
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class B, class C>
+  _CCCL_API auto operator()(B b, C c) const
+  {
+    return b - c - c;
+  }
+};
+struct nstream_op
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class A, class B, class C>
+  _CCCL_API auto operator()(A a, B b, C c) const
+  {
+    return a + b - c - c;
+  }
+};
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+CUB_NAMESPACE_BEGIN
+namespace detail::transform::tile
+{
+template <class T>
+inline constexpr bool tile_eligible_v<mul_op, T, 1> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<add_op, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<triad_op, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<nstream_op, T, 3> = true;
+template <>
+struct tile_operator<mul_op>
+{
+  using type = mul_op;
+};
+template <>
+struct tile_operator<add_op>
+{
+  using type = add_op;
+};
+template <>
+struct tile_operator<triad_op>
+{
+  using type = triad_op;
+};
+template <>
+struct tile_operator<nstream_op>
+{
+  using type = nstream_op;
+};
+} // namespace detail::transform::tile
+CUB_NAMESPACE_END
+#endif // _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+
+// The tile path does not support __int128 (no tensor_span/partition_view for it), so the type axis
+// omits it relative to the base babelstream bench.
+#ifdef TUNE_T
+using element_types = nvbench::type_list<TUNE_T>;
+#else
+using element_types = nvbench::type_list<nvbench::int8_t, nvbench::int16_t, nvbench::float32_t, nvbench::float64_t>;
+#endif
+
+inline auto array_size_powers = nvbench::range(16, 32, 4);
+
+// Same constant inputs as the base bench so nstream maintains a consistent workload.
+inline constexpr auto startA      = 11;
+inline constexpr auto startB      = 2;
+inline constexpr auto startC      = 1;
+inline constexpr auto startScalar = -2;
+static_assert(startA == (startA + startB + startScalar * startC), "nstream must have a consistent workload");
+
+template <typename T>
+static void mul(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(state, cuda::std::tuple{c.begin() + unaligned}, b.begin() + unaligned, n, mul_op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(mul, NVBENCH_TYPE_AXES(element_types))
+  .set_name("tile_mul")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);
+
+template <typename T>
+static void add(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+  thrust::device_vector<T> a(n + unaligned, startA);
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(
+    state, cuda::std::tuple{a.begin() + unaligned, b.begin() + unaligned}, c.begin() + unaligned, n, add_op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(add, NVBENCH_TYPE_AXES(element_types))
+  .set_name("tile_add")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);
+
+template <typename T>
+static void triad(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+  thrust::device_vector<T> a(n + unaligned, startA);
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(
+    state, cuda::std::tuple{b.begin() + unaligned, c.begin() + unaligned}, a.begin() + unaligned, n, triad_op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(triad, NVBENCH_TYPE_AXES(element_types))
+  .set_name("tile_triad")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);
+
+template <typename T>
+static void nstream(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+  thrust::device_vector<T> a(n + unaligned, startA);
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(3 * n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(
+    state,
+    cuda::std::tuple{a.begin() + unaligned, b.begin() + unaligned, c.begin() + unaligned},
+    a.begin() + unaligned,
+    n,
+    nstream_op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(nstream, NVBENCH_TYPE_AXES(element_types))
+  .set_name("tile_nstream")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);

cccl_upstream/cub/benchmarks/bench/transform/tile/copy.cu

@@ -0,0 +1,69 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Pure copy (identity transform) -- measures plain load/store bandwidth through the tile
+// load_masked/store_masked path. The identity op registers a tile_operator substitute (gated); under
+// --enable-tile + CCCL_ENABLE_EXPERIMENTAL_TILE_TRANSFORM_DISPATCH the dispatch hook routes it to the tile kernel,
+// otherwise it falls through to CUB's standard transform. This file disappears once tile dispatch is
+// fully transparent.
+
+#include "../common.h"
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+#  include <cuda_tile.h>
+#endif
+
+struct identity
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class T>
+  _CCCL_API auto operator()(T v) const
+  {
+    return v;
+  }
+};
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+CUB_NAMESPACE_BEGIN
+namespace detail::transform::tile
+{
+template <class T>
+inline constexpr bool tile_eligible_v<identity, T, 1> = true;
+template <>
+struct tile_operator<identity>
+{
+  using type = identity;
+};
+} // namespace detail::transform::tile
+CUB_NAMESPACE_END
+#endif // _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+
+#ifdef TUNE_T
+using element_types = nvbench::type_list<TUNE_T>;
+#else
+using element_types = nvbench::type_list<nvbench::int8_t, nvbench::int16_t, nvbench::int32_t, nvbench::float64_t>;
+#endif
+
+template <typename T>
+static void copy(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+
+  thrust::device_vector<T> in = generate(n);
+  thrust::device_vector<T> out(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(state, cuda::std::tuple{in.begin()}, out.begin(), n, identity{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(copy, NVBENCH_TYPE_AXES(element_types))
+  .set_name("tile_copy")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/tile/grayscale.cu

@@ -0,0 +1,73 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Tile variant of the grayscale transform bench. Unlike the base bench (a single rgb_t<T> struct
+// input), this uses three separate R/G/B streams so the inputs are plain element types the tile path
+// can vectorize. The named rgb_to_y op registers a tile_operator substitute (gated). This file
+// disappears once tile dispatch is fully transparent.
+
+#include "../common.h"
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+#  include <cuda_tile.h>
+#endif
+
+struct rgb_to_y
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class R, class G, class B>
+  _CCCL_API auto operator()(R r, G g, B b) const
+  {
+    constexpr float w_r = 0.2989f;
+    constexpr float w_g = 0.587f;
+    constexpr float w_b = 0.114f;
+    return w_r * r + w_g * g + w_b * b;
+  }
+};
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+CUB_NAMESPACE_BEGIN
+namespace detail::transform::tile
+{
+template <class T>
+inline constexpr bool tile_eligible_v<rgb_to_y, T, 3> = true;
+template <>
+struct tile_operator<rgb_to_y>
+{
+  using type = rgb_to_y;
+};
+} // namespace detail::transform::tile
+CUB_NAMESPACE_END
+#endif // _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+
+#ifdef TUNE_T
+using value_types = nvbench::type_list<TUNE_T>;
+#else
+using value_types = nvbench::type_list<nvbench::float32_t, nvbench::float64_t>;
+#endif
+
+template <typename T>
+static void grayscale(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+
+  thrust::device_vector<T> r = generate(n);
+  thrust::device_vector<T> g = generate(n);
+  thrust::device_vector<T> b = generate(n);
+  thrust::device_vector<T> out(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(3 * n); // matches the base bench's rgb_t<T> = 3 * sizeof(T)
+  state.add_global_memory_writes<T>(n);
+  bench_transform(state, cuda::std::tuple{r.begin(), g.begin(), b.begin()}, out.begin(), n, rgb_to_y{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(grayscale, NVBENCH_TYPE_AXES(value_types))
+  .set_name("tile_grayscale")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/tile/pytorch.cu

@@ -0,0 +1,493 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Tile variant of the PyTorch-style transform benches. Each named op registers a tile_operator
+// substitute (gated); MUFU-heavy ops also opt into tile_mufu_heavy_v so the tile policy picker caps
+// items/thread at the vector width on sub-4-byte types. Under --enable-tile +
+// CCCL_ENABLE_EXPERIMENTAL_TILE_TRANSFORM_DISPATCH the dispatch hook routes them to the tile kernel; otherwise this
+// is the standard CUB path. This file disappears once tile dispatch is fully transparent.
+
+#include <cuda_bf16.h>
+#include <cuda_fp16.h>
+
+#include <cuda/std/cmath>
+
+#include "../common.h"
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+#  include <cuda_tile.h>
+#endif
+
+// Scalar ops the user passes to Transform. Sub-4-byte input types compute in float and cast back,
+// matching the tile substitutes below.
+template <class T>
+__host__ __device__ float to_f(T v)
+{
+  return static_cast<float>(v);
+}
+template <class T>
+__host__ __device__ T from_f(float f)
+{
+  return static_cast<T>(f);
+}
+
+struct relu_op
+{
+  template <class T>
+  __host__ __device__ T operator()(T v) const
+  {
+    float f = to_f(v);
+    return from_f<T>(f > 0.0f ? f : 0.0f);
+  }
+};
+struct sigmoid_op
+{
+  template <class T>
+  __host__ __device__ T operator()(T v) const
+  {
+    float f = to_f(v);
+    return from_f<T>(1.0f / (1.0f + ::cuda::std::exp(-f)));
+  }
+};
+struct tanh_op
+{
+  template <class T>
+  __host__ __device__ T operator()(T v) const
+  {
+    return from_f<T>(::cuda::std::tanh(to_f(v)));
+  }
+};
+struct gelu_op
+{
+  template <class T>
+  __host__ __device__ T operator()(T v) const
+  {
+    constexpr float k0 = 0.7978845608028654f, k1 = 0.044715f;
+    float f = to_f(v);
+    return from_f<T>(0.5f * f * (1.0f + ::cuda::std::tanh(k0 * (f + k1 * f * f * f))));
+  }
+};
+struct sin_op
+{
+  template <class T>
+  __host__ __device__ T operator()(T v) const
+  {
+    return from_f<T>(::cuda::std::sin(to_f(v)));
+  }
+};
+struct exp_op
+{
+  template <class T>
+  __host__ __device__ T operator()(T v) const
+  {
+    return from_f<T>(::cuda::std::exp(to_f(v)));
+  }
+};
+
+struct binary_add
+{
+  template <class A, class B>
+  __host__ __device__ auto operator()(A a, B b) const
+  {
+    return a + b;
+  }
+};
+struct binary_sub
+{
+  template <class A, class B>
+  __host__ __device__ auto operator()(A a, B b) const
+  {
+    return a - b;
+  }
+};
+struct binary_mul
+{
+  template <class A, class B>
+  __host__ __device__ auto operator()(A a, B b) const
+  {
+    return a * b;
+  }
+};
+struct binary_div
+{
+  template <class A, class B>
+  __host__ __device__ auto operator()(A a, B b) const
+  {
+    return a / b;
+  }
+};
+struct binary_le
+{
+  template <class A, class B>
+  __host__ __device__ A operator()(A a, B b) const
+  {
+    return static_cast<A>(a <= b);
+  }
+};
+struct binary_ge
+{
+  template <class A, class B>
+  __host__ __device__ A operator()(A a, B b) const
+  {
+    return static_cast<A>(a >= b);
+  }
+};
+struct binary_fmin
+{
+  template <class A, class B>
+  __host__ __device__ auto operator()(A a, B b) const
+  {
+    return a < b ? a : b;
+  }
+};
+struct binary_fmax
+{
+  template <class A, class B>
+  __host__ __device__ auto operator()(A a, B b) const
+  {
+    return a > b ? a : b;
+  }
+};
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+namespace ct = ::cuda::tiles;
+
+template <class T>
+__tile__ auto as_float(T v)
+{
+  return ct::element_cast<float>(v);
+}
+template <class T, class F>
+__tile__ auto from_float(F f)
+{
+  return ct::element_cast<ct::tile_element_t<T>>(f);
+}
+
+struct tile_relu
+{
+  template <class T>
+  __tile__ auto operator()(T v) const
+  {
+    auto f = as_float(v);
+    return from_float<T>(ct::select(f > 0.0f, f, f - f));
+  }
+};
+struct tile_sigmoid
+{
+  template <class T>
+  __tile__ auto operator()(T v) const
+  {
+    auto f = as_float(v);
+    return from_float<T>(1.0f / (1.0f + ct::exp(-f)));
+  }
+};
+struct tile_tanh
+{
+  template <class T>
+  __tile__ auto operator()(T v) const
+  {
+    return from_float<T>(ct::tanh(as_float(v)));
+  }
+};
+struct tile_gelu
+{
+  template <class T>
+  __tile__ auto operator()(T v) const
+  {
+    constexpr float k0 = 0.7978845608028654f, k1 = 0.044715f;
+    auto f = as_float(v);
+    return from_float<T>(0.5f * f * (1.0f + ct::tanh(k0 * (f + k1 * f * f * f))));
+  }
+};
+struct tile_sin
+{
+  template <class T>
+  __tile__ auto operator()(T v) const
+  {
+    return from_float<T>(ct::sin(as_float(v)));
+  }
+};
+struct tile_exp
+{
+  template <class T>
+  __tile__ auto operator()(T v) const
+  {
+    return from_float<T>(ct::exp(as_float(v)));
+  }
+};
+
+struct tile_binary_add
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return a + b;
+  }
+};
+struct tile_binary_sub
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return a - b;
+  }
+};
+struct tile_binary_mul
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return a * b;
+  }
+};
+struct tile_binary_div
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return a / b;
+  }
+};
+struct tile_binary_le
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return ct::element_cast<ct::tile_element_t<A>>(a <= b);
+  }
+};
+struct tile_binary_ge
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return ct::element_cast<ct::tile_element_t<A>>(a >= b);
+  }
+};
+struct tile_binary_fmin
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return ct::select(a < b, a, b);
+  }
+};
+struct tile_binary_fmax
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return ct::select(a > b, a, b);
+  }
+};
+
+CUB_NAMESPACE_BEGIN
+namespace detail::transform::tile
+{
+// Unary
+template <class T>
+inline constexpr bool tile_eligible_v<relu_op, T, 1> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<sigmoid_op, T, 1> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<tanh_op, T, 1> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<gelu_op, T, 1> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<sin_op, T, 1> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<exp_op, T, 1> = true;
+template <>
+struct tile_operator<relu_op>
+{
+  using type = tile_relu;
+};
+template <>
+struct tile_operator<sigmoid_op>
+{
+  using type = tile_sigmoid;
+};
+template <>
+struct tile_operator<tanh_op>
+{
+  using type = tile_tanh;
+};
+template <>
+struct tile_operator<gelu_op>
+{
+  using type = tile_gelu;
+};
+template <>
+struct tile_operator<sin_op>
+{
+  using type = tile_sin;
+};
+template <>
+struct tile_operator<exp_op>
+{
+  using type = tile_exp;
+};
+
+// MUFU-heavy unary ops: hint the tile policy picker to cap items/thread at the vector width on
+// sub-4-byte types.
+template <>
+inline constexpr bool tile_mufu_heavy_v<sigmoid_op> = true;
+template <>
+inline constexpr bool tile_mufu_heavy_v<tanh_op> = true;
+template <>
+inline constexpr bool tile_mufu_heavy_v<gelu_op> = true;
+template <>
+inline constexpr bool tile_mufu_heavy_v<sin_op> = true;
+template <>
+inline constexpr bool tile_mufu_heavy_v<exp_op> = true;
+
+// Binary
+template <class T>
+inline constexpr bool tile_eligible_v<binary_add, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_sub, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_mul, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_div, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_le, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_ge, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_fmin, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_fmax, T, 2> = true;
+template <>
+struct tile_operator<binary_add>
+{
+  using type = tile_binary_add;
+};
+template <>
+struct tile_operator<binary_sub>
+{
+  using type = tile_binary_sub;
+};
+template <>
+struct tile_operator<binary_mul>
+{
+  using type = tile_binary_mul;
+};
+template <>
+struct tile_operator<binary_div>
+{
+  using type = tile_binary_div;
+};
+template <>
+struct tile_operator<binary_le>
+{
+  using type = tile_binary_le;
+};
+template <>
+struct tile_operator<binary_ge>
+{
+  using type = tile_binary_ge;
+};
+template <>
+struct tile_operator<binary_fmin>
+{
+  using type = tile_binary_fmin;
+};
+template <>
+struct tile_operator<binary_fmax>
+{
+  using type = tile_binary_fmax;
+};
+} // namespace detail::transform::tile
+CUB_NAMESPACE_END
+#endif // _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+
+#ifdef TUNE_T
+using element_types = nvbench::type_list<TUNE_T>;
+#else
+using element_types = nvbench::type_list<
+#  if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
+  __half,
+#  endif
+#  if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
+  __nv_bfloat16,
+#  endif
+  nvbench::float32_t>;
+#endif
+
+template <typename Op, typename T>
+static void run_unary(nvbench::state& state)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+  thrust::device_vector<T> in(n, T(1));
+  thrust::device_vector<T> out(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(state, cuda::std::tuple{in.begin()}, out.begin(), n, Op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+template <typename Op, typename T>
+static void run_binary(nvbench::state& state)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+  thrust::device_vector<T> a(n, T(1));
+  thrust::device_vector<T> b(n, T(1));
+  thrust::device_vector<T> out(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(state, cuda::std::tuple{a.begin(), b.begin()}, out.begin(), n, Op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+inline auto pt_sizes = nvbench::range(16, 32, 4);
+
+#define UNARY_BENCH(name, op)                                            \
+  template <typename T>                                                  \
+  static void name##_bench(nvbench::state& state, nvbench::type_list<T>) \
+  {                                                                      \
+    run_unary<op, T>(state);                                             \
+  }                                                                      \
+  NVBENCH_BENCH_TYPES(name##_bench, NVBENCH_TYPE_AXES(element_types))    \
+    .set_name("tile_" #name)                                             \
+    .set_type_axes_names({"T{ct}"})                                      \
+    .add_int64_power_of_two_axis("Elements{io}", pt_sizes)
+
+UNARY_BENCH(relu, relu_op);
+UNARY_BENCH(sigmoid, sigmoid_op);
+UNARY_BENCH(tanh, tanh_op);
+UNARY_BENCH(gelu, gelu_op);
+UNARY_BENCH(sin, sin_op);
+UNARY_BENCH(exp, exp_op);
+
+#define BINARY_BENCH(name, op)                                           \
+  template <typename T>                                                  \
+  static void name##_bench(nvbench::state& state, nvbench::type_list<T>) \
+  {                                                                      \
+    run_binary<op, T>(state);                                            \
+  }                                                                      \
+  NVBENCH_BENCH_TYPES(name##_bench, NVBENCH_TYPE_AXES(element_types))    \
+    .set_name("tile_pt_" #name)                                          \
+    .set_type_axes_names({"T{ct}"})                                      \
+    .add_int64_power_of_two_axis("Elements{io}", pt_sizes)
+
+BINARY_BENCH(add, binary_add);
+BINARY_BENCH(sub, binary_sub);
+BINARY_BENCH(mul, binary_mul);
+BINARY_BENCH(div, binary_div);
+BINARY_BENCH(le, binary_le);
+BINARY_BENCH(ge, binary_ge);
+BINARY_BENCH(fmin, binary_fmin);
+BINARY_BENCH(fmax, binary_fmax);

cccl_upstream/cub/benchmarks/bench/transform_reduce/sum.cu

@@ -0,0 +1,83 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_reduce.cuh>
+
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_ITEMS_PER_VEC_LOAD_POW2 ipv 1:2:1
+
+#if !TUNE_BASE
+template <typename AccumT>
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::ReducePolicy
+  {
+    const auto [items, threads] =
+      cub::detail::scale_mem_bound(TUNE_THREADS_PER_BLOCK, TUNE_ITEMS_PER_THREAD, int{sizeof(AccumT)});
+    const auto policy = cub::ReducePassPolicy{
+      threads, items, 1 << TUNE_ITEMS_PER_VEC_LOAD_POW2, cub::BLOCK_REDUCE_WARP_REDUCTIONS, cub::LOAD_DEFAULT};
+    return {policy, policy};
+  }
+};
+#endif // !TUNE_BASE
+
+template <class T>
+struct square_t
+{
+  __host__ __device__ T operator()(const T& x) const
+  {
+    return x * x;
+  }
+};
+
+template <typename T, typename OffsetT>
+void reduce(nvbench::state& state, nvbench::type_list<T, OffsetT>)
+{
+  using init_value_t   = T;
+  using reduction_op_t = ::cuda::std::plus<>;
+  using transform_op_t = square_t<T>;
+
+  // Retrieve axis parameters
+  const auto elements = state.get_int64("Elements{io}");
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<T> out(1, thrust::default_init);
+
+  auto d_in  = thrust::raw_pointer_cast(in.data());
+  auto d_out = thrust::raw_pointer_cast(out.data());
+
+  // Enable throughput calculations and add "Size" column to results.
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements, "Size");
+  state.add_global_memory_writes<T>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(policy_selector<cuda::std::__accumulator_t<reduction_op_t, T, init_value_t>>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceReduce::TransformReduce,
+      "TransformReduce failed",
+      d_in,
+      d_out,
+      static_cast<OffsetT>(elements),
+      reduction_op_t{},
+      transform_op_t{},
+      init_value_t{},
+      env);
+  });
+}
+
+NVBENCH_BENCH_TYPES(reduce, NVBENCH_TYPE_AXES(all_types, offset_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/docker/.gitignore

@@ -0,0 +1 @@
+Dockerfile

cccl_upstream/cub/benchmarks/docker/recipe.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+import hpccm
+
+hpccm.config.set_container_format("docker")
+
+Stage0 = hpccm.primitives.baseimage(image="nvidia/cuda:12.2.0-devel-ubuntu22.04")
+Stage0 += hpccm.building_blocks.apt_get(
+    ospackages=[
+        "git",
+        "tmux",
+        "gcc",
+        "g++",
+        "vim",
+        "python3",
+        "python-is-python3",
+        "ninja-build",
+    ]
+)
+# Stage0 += hpccm.building_blocks.llvm(version='15', extra_tools=True, toolset=True)
+Stage0 += hpccm.building_blocks.cmake(eula=True, version="3.26.3")
+# Stage0 += hpccm.building_blocks.nsight_compute(eula=True, version='2023.1.1')
+Stage0 += hpccm.building_blocks.pip(
+    packages=["fpzip", "numpy", "pandas", "pynvml"], pip="pip3"
+)
+Stage0 += hpccm.primitives.environment(variables={"CUDA_MODULE_LOADING": "EAGER"})