[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/thrust/benchmarks/CMakeLists.txt

@@ -0,0 +1,91 @@
+include(${CMAKE_SOURCE_DIR}/benchmarks/cmake/CCCLBenchmarkRegistry.cmake)
+
+cccl_get_nvbench()
+cccl_get_nvbench_helper()
+
+set(benches_root "${CMAKE_CURRENT_LIST_DIR}")
+
+function(get_recursive_subdirs subdirs)
+  set(dirs)
+  file(
+    GLOB_RECURSE contents
+    CONFIGURE_DEPENDS
+    LIST_DIRECTORIES ON
+    "${CMAKE_CURRENT_LIST_DIR}/bench/*"
+  )
+
+  foreach (test_dir IN LISTS contents)
+    if (IS_DIRECTORY "${test_dir}")
+      list(APPEND dirs "${test_dir}")
+    endif()
+  endforeach()
+
+  set(${subdirs} "${dirs}" PARENT_SCOPE)
+endfunction()
+
+function(add_bench target_name bench_name bench_src)
+  set(bench_target ${bench_name})
+  set(${target_name} ${bench_target} PARENT_SCOPE)
+
+  cccl_add_executable(${bench_target} SOURCES "${bench_src}")
+  target_link_libraries(
+    ${bench_target}
+    PRIVATE #
+      cccl.nvbench_helper
+      nvbench::main
+  )
+endfunction()
+
+function(thrust_wrap_bench_in_cpp cpp_file_var cu_file thrust_target)
+  thrust_get_target_property(prefix ${thrust_target} PREFIX)
+  set(wrapped_source_file "${cu_file}")
+  set(cpp_file "${CMAKE_CURRENT_BINARY_DIR}/${prefix}/${cu_file}.cpp")
+  configure_file(
+    "${Thrust_SOURCE_DIR}/cmake/wrap_source_file.cpp.in"
+    "${cpp_file}"
+  )
+  set(${cpp_file_var} "${cpp_file}" PARENT_SCOPE)
+endfunction()
+
+function(add_bench_dir bench_dir)
+  file(GLOB bench_srcs CONFIGURE_DEPENDS "${bench_dir}/*.cu")
+  file(RELATIVE_PATH bench_prefix "${benches_root}" "${bench_dir}")
+  file(TO_CMAKE_PATH "${bench_prefix}" bench_prefix)
+  string(REPLACE "/" "." bench_prefix "${bench_prefix}")
+
+  foreach (bench_src IN LISTS bench_srcs)
+    foreach (thrust_target IN LISTS THRUST_TARGETS)
+      thrust_get_target_property(config_prefix ${thrust_target} PREFIX)
+      thrust_get_target_property(config_device ${thrust_target} DEVICE)
+
+      # Wrap the .cu file in .cpp for non-CUDA backends
+      if ("CUDA" STREQUAL "${config_device}")
+        set(real_bench_src "${bench_src}")
+      else()
+        thrust_wrap_bench_in_cpp(real_bench_src "${bench_src}" ${thrust_target})
+      endif()
+
+      get_filename_component(bench_name "${bench_src}" NAME_WLE)
+      string(PREPEND bench_name "${config_prefix}.${bench_prefix}.")
+      register_cccl_benchmark("${bench_name}" "")
+
+      string(APPEND bench_name ".base")
+      add_bench(base_bench_target ${bench_name} "${real_bench_src}")
+      cccl_configure_target(${bench_name})
+      target_link_libraries(${bench_name} PRIVATE ${thrust_target})
+
+      if ("CUDA" STREQUAL "${config_device}")
+        target_compile_options(
+          ${bench_name}
+          PRIVATE "$<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--extended-lambda>"
+        )
+      endif()
+    endforeach()
+  endforeach()
+endfunction()
+
+get_recursive_subdirs(subdirs)
+
+foreach (subdir IN LISTS subdirs)
+  add_bench_dir("${subdir}")
+endforeach()

cccl_upstream/thrust/benchmarks/bench/adjacent_difference/basic.cu

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/adjacent_difference.h>
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+    thrust::adjacent_difference(policy(alloc, launch), input.cbegin(), input.cend(), output.begin());
+  });
+}
+
+using types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double>;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/adjacent_difference/custom.cu

@@ -0,0 +1,52 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/adjacent_difference.h>
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+struct custom_op
+{
+  T val;
+
+  custom_op() = delete;
+
+  explicit custom_op(T val)
+      : val(val)
+  {}
+
+  __device__ T operator()(const T& lhs, const T& rhs)
+  {
+    return lhs * rhs + val; // Hope to gen mad
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::adjacent_difference(
+                 policy(alloc, launch), input.cbegin(), input.cend(), output.begin(), custom_op<T>{42});
+             });
+}
+
+using types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double>;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/adjacent_difference/in_place.cu

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/adjacent_difference.h>
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> vec(elements, 0);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::adjacent_difference(policy(alloc, launch), vec.begin(), vec.end(), vec.begin());
+             });
+}
+
+using types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double>;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/all_of/basic.cu

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/fill.h>
+#include <thrust/logical.h>
+
+#include <cuda/functional>
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  T val = 1;
+  // set up input
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto common_prefix  = state.get_float64("MismatchAt");
+  const auto mismatch_point = static_cast<std::size_t>(static_cast<double>(elements) * common_prefix);
+
+  thrust::device_vector<T> dinput(elements, thrust::no_init);
+  thrust::fill(dinput.begin(), dinput.begin() + mismatch_point, T{0});
+  thrust::fill(dinput.begin() + mismatch_point, dinput.end(), val);
+
+  state.add_global_memory_reads<T>(mismatch_point + 1);
+  state.add_global_memory_writes<size_t>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      do_not_optimize(thrust::all_of(policy(alloc, launch), dinput.begin(), dinput.end(), cuda::equal_to_value{val}));
+    });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_float64_axis("MismatchAt", std::vector{1.0, 0.5, 0.01});

cccl_upstream/thrust/benchmarks/bench/any_of/basic.cu

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/fill.h>
+#include <thrust/logical.h>
+
+#include <cuda/functional>
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  T val = 1;
+  // set up input
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto common_prefix  = state.get_float64("MismatchAt");
+  const auto mismatch_point = static_cast<std::size_t>(static_cast<double>(elements) * common_prefix);
+
+  thrust::device_vector<T> dinput(elements, thrust::no_init);
+  thrust::fill(dinput.begin(), dinput.begin() + mismatch_point, T{0});
+  thrust::fill(dinput.begin() + mismatch_point, dinput.end(), val);
+
+  state.add_global_memory_reads<T>(mismatch_point + 1);
+  state.add_global_memory_writes<size_t>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      do_not_optimize(thrust::any_of(policy(alloc, launch), dinput.begin(), dinput.end(), cuda::equal_to_value{val}));
+    });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_float64_axis("MismatchAt", std::vector{1.0, 0.5, 0.01});

cccl_upstream/thrust/benchmarks/bench/copy/basic.cu

@@ -0,0 +1,68 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/copy.h>
+#include <thrust/count.h>
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+
+#include <nvbench_helper.cuh>
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> input(elements, T{1});
+  thrust::device_vector<T> output(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::copy(policy(alloc, launch), input.cbegin(), input.cend(), output.begin());
+             });
+}
+
+// Non-trivially-copyable/relocatable type which is not allowed to be copied using std::memcpy or cudaMemcpy
+struct non_trivial
+{
+  int a;
+  int b;
+
+  non_trivial() = default;
+
+  _CCCL_HOST_DEVICE explicit non_trivial(int i)
+      : a(i)
+      , b(i)
+  {}
+
+  // the user-defined copy constructor prevents the type from being trivially copyable
+  // NOLINTNEXTLINE(modernize-use-equals-default)
+  _CCCL_HOST_DEVICE non_trivial(const non_trivial& nt)
+      : a(nt.a)
+      , b(nt.b)
+  {}
+
+  // NOLINTNEXTLINE(modernize-use-equals-default)
+  _CCCL_HOST_DEVICE non_trivial& operator=(const non_trivial& nt)
+  {
+    a = nt.a;
+    b = nt.b;
+    return *this;
+  }
+};
+
+static_assert(!::cuda::std::is_trivially_copyable<non_trivial>::value); // as required by the C++ standard
+static_assert(!thrust::is_trivially_relocatable<non_trivial>::value); // thrust uses this check internally
+
+using types =
+  nvbench::type_list<nvbench::uint8_t, nvbench::uint16_t, nvbench::uint32_t, nvbench::uint64_t, non_trivial>;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/copy/if.cu

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/copy.h>
+#include <thrust/count.h>
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+
+#include <nvbench_helper.cuh>
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  using select_op_t = less_then_t<T>;
+
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  const T val = lerp_min_max<T>(entropy_to_probability(entropy));
+  select_op_t select_op{val};
+
+  thrust::device_vector<T> input = generate(elements);
+  const auto selected_elements   = thrust::count_if(input.cbegin(), input.cend(), select_op);
+  thrust::device_vector<T> output(selected_elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(selected_elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::copy_if(policy(alloc, launch), input.cbegin(), input.cend(), output.begin(), select_op);
+             });
+}
+
+using types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t>;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.000"});

cccl_upstream/thrust/benchmarks/bench/copy_if/basic.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/copy.h>
+#include <thrust/device_vector.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+struct is_even
+{
+  template <class T>
+  __device__ constexpr bool operator()(const T& val) const noexcept
+  {
+    return static_cast<int>(val) % 2 == 0;
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements, bit_entropy::_1_000, T{0}, T{42});
+  thrust::device_vector<T> out(elements, thrust::no_init);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(thrust::copy_if(policy(alloc, launch), in.begin(), in.end(), out.begin(), is_even{}));
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/count/basic.cu

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/count.h>
+#include <thrust/device_vector.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+    do_not_optimize(thrust::count(policy(alloc, launch), in.begin(), in.end(), T{42}));
+  });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/count_if/basic.cu

@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/count.h>
+#include <thrust/device_vector.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+struct equal_to_42
+{
+  template <class T>
+  __device__ constexpr bool operator()(const T& val) const noexcept
+  {
+    return val == 42;
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+    do_not_optimize(thrust::count_if(policy(alloc, launch), in.begin(), in.end(), equal_to_42{}));
+  });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/equal/basic.cu

@@ -0,0 +1,38 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/equal.h>
+#include <thrust/execution_policy.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void benchmark(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> a(elements, T{1});
+  thrust::device_vector<T> b(elements, T{1});
+
+  const auto common_prefix = state.get_float64("CommonPrefixRatio");
+  const auto same_elements =
+    std::min(static_cast<std::size_t>(static_cast<double>(elements) * common_prefix), elements);
+  caching_allocator_t alloc;
+  thrust::fill(policy(alloc), b.begin() + same_elements, b.end(), T{2});
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(2 * std::max(same_elements, std::size_t(1))); // using `same_elements` instead
+                                                                                 // of `elements` corresponds to the
+                                                                                 // actual elements read in an early
+                                                                                 // exit
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(thrust::equal(policy(alloc, launch), a.begin(), a.end(), b.begin()));
+             });
+}
+
+NVBENCH_BENCH_TYPES(benchmark, NVBENCH_TYPE_AXES(integral_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_float64_axis("CommonPrefixRatio", std::vector{1.0, 0.5, 0.0});

cccl_upstream/thrust/benchmarks/bench/extrema/basic.cu

@@ -0,0 +1,54 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/extrema.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T, typename Func>
+static void bench_extremum(nvbench::state& state, nvbench::type_list<T>, Func func)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements);
+
+  using offset_t = typename decltype(in.cbegin())::difference_type;
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<offset_t>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(func(policy(alloc, launch), in.cbegin(), in.cend()));
+             });
+}
+
+template <typename T>
+static void min_element(nvbench::state& state, nvbench::type_list<T> list)
+{
+  bench_extremum(state, list, [](auto&&... args) {
+    return thrust::min_element(args...);
+  });
+}
+
+NVBENCH_BENCH_TYPES(min_element, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("min_element")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));
+
+template <typename T>
+static void max_element(nvbench::state& state, nvbench::type_list<T> list)
+{
+  bench_extremum(state, list, [](auto&&... args) {
+    return thrust::max_element(args...);
+  });
+}
+
+NVBENCH_BENCH_TYPES(max_element, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("max_element")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/fill/basic.cu

@@ -0,0 +1,30 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/fill.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> output(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::fill(policy(alloc, launch), output.begin(), output.end(), T{42});
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/fill_n/basic.cu

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/fill.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> output(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(thrust::fill_n(policy(alloc, launch), output.begin(), elements, T{42}));
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/find/basic.cu

@@ -0,0 +1,39 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/find.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  T val = 1;
+  // set up input
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto common_prefix  = state.get_float64("MismatchAt");
+  const auto mismatch_point = static_cast<std::size_t>(static_cast<double>(elements) * common_prefix);
+
+  thrust::device_vector<T> dinput(elements, thrust::no_init);
+  thrust::fill(dinput.begin(), dinput.begin() + mismatch_point, T{0});
+  thrust::fill(dinput.begin() + mismatch_point, dinput.end(), val);
+
+  state.add_global_memory_reads<T>(mismatch_point + 1);
+  state.add_global_memory_writes<size_t>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(thrust::find(policy(alloc, launch), dinput.begin(), dinput.end(), val));
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_float64_axis("MismatchAt", std::vector{1.0, 0.5, 0.01});

cccl_upstream/thrust/benchmarks/bench/find_if/basic.cu

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/find.h>
+
+#include <cuda/functional>
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  T val = 1;
+  // set up input
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto common_prefix  = state.get_float64("MismatchAt");
+  const auto mismatch_point = static_cast<std::size_t>(static_cast<double>(elements) * common_prefix);
+
+  thrust::device_vector<T> dinput(elements, thrust::no_init);
+  thrust::fill(dinput.begin(), dinput.begin() + mismatch_point, T{0});
+  thrust::fill(dinput.begin() + mismatch_point, dinput.end(), val);
+
+  state.add_global_memory_reads<T>(mismatch_point + 1);
+  state.add_global_memory_writes<size_t>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(
+                 thrust::find_if(policy(alloc, launch), dinput.begin(), dinput.end(), cuda::equal_to_value<T>{val}));
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_float64_axis("MismatchAt", std::vector{1.0, 0.5, 0.01});

cccl_upstream/thrust/benchmarks/bench/find_if_not/basic.cu

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/find.h>
+
+#include <cuda/functional>
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  T val = 1;
+  // set up input
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto common_prefix  = state.get_float64("MismatchAt");
+  const auto mismatch_point = static_cast<std::size_t>(static_cast<double>(elements) * common_prefix);
+
+  thrust::device_vector<T> dinput(elements, thrust::no_init);
+  thrust::fill(dinput.begin(), dinput.begin() + mismatch_point, T{0});
+  thrust::fill(dinput.begin() + mismatch_point, dinput.end(), val);
+
+  state.add_global_memory_reads<T>(mismatch_point + 1);
+  state.add_global_memory_writes<size_t>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(thrust::find_if_not(
+                 policy(alloc, launch), dinput.begin(), dinput.end(), cuda::std::not_fn(cuda::equal_to_value{val})));
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_float64_axis("MismatchAt", std::vector{1.0, 0.5, 0.01});

cccl_upstream/thrust/benchmarks/bench/for_each/basic.cu

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/for_each.h>
+
+#include "nvbench_helper.cuh"
+
+template <class T>
+struct square_t
+{
+  __device__ void operator()(T& x) const
+  {
+    x = x * x;
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in(elements, T{1});
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  square_t<T> op{};
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::for_each(policy(alloc, launch), in.begin(), in.end(), op);
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/for_each_n/basic.cu

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/for_each.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <class T>
+struct square_t
+{
+  __device__ void operator()(T& x) const
+  {
+    x = x * x;
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in(elements, T{1});
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  square_t<T> op{};
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::for_each_n(policy(alloc, launch), in.begin(), elements, op);
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/generate/basic.cu

@@ -0,0 +1,39 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/generate.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+struct generator
+{
+  _CCCL_DEVICE_API _CCCL_FORCEINLINE auto operator()() const -> T
+  {
+    return 42;
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> output(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::generate(policy(alloc, launch), output.begin(), output.end(), generator<T>{});
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/generate_n/basic.cu

@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/generate.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+struct generator
+{
+  _CCCL_DEVICE_API _CCCL_FORCEINLINE auto operator()() const -> T
+  {
+    return 42;
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> output(elements, thrust::no_init);
+
+  state.add_element_count(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::generate_n(policy(alloc, launch), output.begin(), elements, generator<T>{});
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/inner_product/basic.cu

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/inner_product.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  auto generator               = generate(elements);
+  thrust::device_vector<T> lhs = generator;
+  thrust::device_vector<T> rhs = generator;
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements * 2);
+  state.add_global_memory_writes<T>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::inner_product(policy(alloc, launch), lhs.begin(), lhs.end(), rhs.begin(), T{0});
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(all_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/is_partitioned/basic.cu

@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/partition.h>
+#include <thrust/sequence.h>
+
+#include <cuda/functional>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  using select_op_t = less_then_t<T>;
+
+  // set up input
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto common_prefix  = state.get_float64("MismatchAt");
+  const auto mismatch_point = static_cast<std::size_t>(
+    ::cuda::std::clamp(static_cast<double>(elements) * common_prefix, 0.0, static_cast<double>(elements - 1)));
+
+  thrust::device_vector<T> dinput(elements, thrust::no_init);
+  thrust::sequence(dinput.begin(), dinput.end(), T{0});
+
+  state.add_global_memory_reads<T>(2 * elements);
+  state.add_global_memory_writes<size_t>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(thrust::is_partitioned(
+                 policy(alloc, launch), dinput.begin(), dinput.end(), select_op_t{static_cast<T>(mismatch_point)}));
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_float64_axis("MismatchAt", std::vector{1.0, 0.5, 0.01});

cccl_upstream/thrust/benchmarks/bench/is_sorted/basic.cu

@@ -0,0 +1,69 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/sequence.h>
+#include <thrust/sort.h>
+
+#include <cuda/functional>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  // set up input
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto common_prefix  = state.get_float64("MismatchAt");
+  const auto mismatch_point = static_cast<std::size_t>(
+    ::cuda::std::clamp(static_cast<double>(elements) * common_prefix, 0.0, static_cast<double>(elements - 1)));
+
+  thrust::device_vector<T> dinput(elements, thrust::no_init);
+  thrust::sequence(dinput.begin(), dinput.end(), T{0});
+  dinput[mismatch_point] = T{-1};
+
+  state.add_global_memory_reads<T>(mismatch_point + 1);
+  state.add_global_memory_writes<size_t>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(thrust::is_sorted(policy(alloc, launch), dinput.begin(), dinput.end()));
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_float64_axis("MismatchAt", std::vector{1.0, 0.5, 0.01});
+
+template <typename T>
+static void with_predicate(nvbench::state& state, nvbench::type_list<T>)
+{
+  // set up input
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto common_prefix  = state.get_float64("MismatchAt");
+  const auto mismatch_point = static_cast<std::size_t>(
+    ::cuda::std::clamp(static_cast<double>(elements) * common_prefix, 0.0, static_cast<double>(elements - 1)));
+
+  thrust::device_vector<T> dinput(elements, thrust::no_init);
+  thrust::sequence(dinput.begin(), dinput.end(), T{0});
+  dinput[mismatch_point] = T{-1};
+
+  state.add_global_memory_reads<T>(mismatch_point + 1);
+  state.add_global_memory_writes<size_t>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      do_not_optimize(thrust::is_sorted(policy(alloc, launch), dinput.begin(), dinput.end(), cuda::std::less<>{}));
+    });
+}
+
+NVBENCH_BENCH_TYPES(with_predicate, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("with_predicate")
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_float64_axis("MismatchAt", std::vector{1.0, 0.5, 0.01});

cccl_upstream/thrust/benchmarks/bench/is_sorted_until/basic.cu

@@ -0,0 +1,70 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/sequence.h>
+#include <thrust/sort.h>
+
+#include <cuda/functional>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  // set up input
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto common_prefix  = state.get_float64("MismatchAt");
+  const auto mismatch_point = static_cast<std::size_t>(
+    ::cuda::std::clamp(static_cast<double>(elements) * common_prefix, 0.0, static_cast<double>(elements - 1)));
+
+  thrust::device_vector<T> dinput(elements, thrust::no_init);
+  thrust::sequence(dinput.begin(), dinput.end(), T{0});
+  dinput[mismatch_point] = T{-1};
+
+  state.add_global_memory_reads<T>(mismatch_point + 1);
+  state.add_global_memory_writes<size_t>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(thrust::is_sorted_until(policy(alloc, launch), dinput.begin(), dinput.end()));
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_float64_axis("MismatchAt", std::vector{1.0, 0.5, 0.01});
+
+template <typename T>
+static void with_predicate(nvbench::state& state, nvbench::type_list<T>)
+{
+  // set up input
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto common_prefix  = state.get_float64("MismatchAt");
+  const auto mismatch_point = static_cast<std::size_t>(
+    ::cuda::std::clamp(static_cast<double>(elements) * common_prefix, 0.0, static_cast<double>(elements - 1)));
+
+  thrust::device_vector<T> dinput(elements, thrust::no_init);
+  thrust::sequence(dinput.begin(), dinput.end(), T{0});
+  dinput[mismatch_point] = T{-1};
+
+  state.add_global_memory_reads<T>(mismatch_point + 1);
+  state.add_global_memory_writes<size_t>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(
+                 thrust::is_sorted_until(policy(alloc, launch), dinput.begin(), dinput.end(), cuda::std::less<>{}));
+             });
+}
+
+NVBENCH_BENCH_TYPES(with_predicate, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("with_predicate")
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_float64_axis("MismatchAt", std::vector{1.0, 0.5, 0.01});

cccl_upstream/thrust/benchmarks/bench/merge/basic.cu

@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/merge.h>
+#include <thrust/sort.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements        = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto size_ratio      = static_cast<std::size_t>(state.get_int64("InputSizeRatio"));
+  const auto entropy         = str_to_entropy(state.get_string("Entropy"));
+  const auto elements_in_lhs = static_cast<std::size_t>(static_cast<double>(size_ratio * elements) / 100.0);
+
+  thrust::device_vector<T> out(elements);
+  thrust::device_vector<T> in = generate(elements, entropy);
+  thrust::sort(in.begin(), in.begin() + elements_in_lhs);
+  thrust::sort(in.begin() + elements_in_lhs, in.end());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::merge(
+                 policy(alloc, launch),
+                 in.cbegin(),
+                 in.cbegin() + elements_in_lhs,
+                 in.cbegin() + elements_in_lhs,
+                 in.cend(),
+                 out.begin());
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("InputSizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/mismatch/basic.cu

@@ -0,0 +1,42 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/fill.h>
+#include <thrust/mismatch.h>
+
+#include <cuda/iterator>
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void range_iter(nvbench::state& state, nvbench::type_list<T>)
+{
+  T val = 1;
+  // set up input
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto common_prefix  = state.get_float64("MismatchAt");
+  const auto mismatch_point = static_cast<std::size_t>(static_cast<double>(elements) * common_prefix);
+
+  thrust::device_vector<T> dinput(elements, thrust::no_init);
+  thrust::fill(dinput.begin(), dinput.begin() + mismatch_point, T{0});
+  thrust::fill(dinput.begin() + mismatch_point, dinput.end(), val);
+
+  state.add_global_memory_reads<T>(mismatch_point + 1);
+  state.add_global_memory_writes<size_t>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(
+                 thrust::mismatch(policy(alloc, launch), dinput.begin(), dinput.end(), cuda::constant_iterator<T>{0}));
+             });
+}
+
+NVBENCH_BENCH_TYPES(range_iter, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base_range_iter")
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_float64_axis("MismatchAt", std::vector{1.0, 0.5, 0.01});

cccl_upstream/thrust/benchmarks/bench/none_of/basic.cu

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/fill.h>
+#include <thrust/logical.h>
+
+#include <cuda/functional>
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  T val = 1;
+  // set up input
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto common_prefix  = state.get_float64("MismatchAt");
+  const auto mismatch_point = static_cast<std::size_t>(static_cast<double>(elements) * common_prefix);
+
+  thrust::device_vector<T> dinput(elements, thrust::no_init);
+  thrust::fill(dinput.begin(), dinput.begin() + mismatch_point, T{0});
+  thrust::fill(dinput.begin() + mismatch_point, dinput.end(), val);
+
+  state.add_global_memory_reads<T>(mismatch_point + 1);
+  state.add_global_memory_writes<size_t>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      do_not_optimize(thrust::none_of(policy(alloc, launch), dinput.begin(), dinput.end(), cuda::equal_to_value{val}));
+    });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_float64_axis("MismatchAt", std::vector{1.0, 0.5, 0.01});

cccl_upstream/thrust/benchmarks/bench/partition/basic.cu

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/partition.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  using select_op_t = less_then_t<T>;
+
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  const T val = lerp_min_max<T>(entropy_to_probability(entropy));
+  select_op_t select_op{val};
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::partition_copy(
+                 policy(alloc, launch),
+                 input.cbegin(),
+                 input.cend(),
+                 output.begin(),
+                 cuda::std::make_reverse_iterator(output.begin() + elements),
+                 select_op);
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.000"});

cccl_upstream/thrust/benchmarks/bench/reduce/basic.cu

@@ -0,0 +1,30 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/reduce.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+    do_not_optimize(thrust::reduce(policy(alloc, launch), in.begin(), in.end()));
+  });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/reduce/by_key.cu

@@ -0,0 +1,60 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/reduce.h>
+#include <thrust/unique.h>
+
+#include "nvbench_helper.cuh"
+
+template <class KeyT, class ValueT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  constexpr std::size_t min_segment_size = 1;
+  const std::size_t max_segment_size     = static_cast<std::size_t>(state.get_int64("MaxSegSize"));
+
+  thrust::device_vector<KeyT> in_keys  = generate.uniform.key_segments(elements, min_segment_size, max_segment_size);
+  thrust::device_vector<KeyT> out_keys = in_keys;
+  thrust::device_vector<ValueT> in_vals(elements);
+
+  const std::size_t unique_keys =
+    ::cuda::std::distance(out_keys.begin(), thrust::unique(out_keys.begin(), out_keys.end()));
+
+  thrust::device_vector<ValueT> out_vals(unique_keys);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+
+  state.add_global_memory_writes<KeyT>(unique_keys);
+  state.add_global_memory_writes<ValueT>(unique_keys);
+
+  caching_allocator_t alloc;
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      thrust::reduce_by_key(
+        policy(alloc, launch), in_keys.begin(), in_keys.end(), in_vals.begin(), out_keys.begin(), out_vals.begin());
+    });
+}
+
+using key_types =
+  nvbench::type_list<int8_t,
+                     int16_t,
+                     int32_t,
+                     int64_t
+#if _CCCL_HAS_INT128()
+                     ,
+                     int128_t
+#endif
+                     >;
+
+using value_types = all_types;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegSize", {1, 4, 8});

cccl_upstream/thrust/benchmarks/bench/remove/basic.cu

@@ -0,0 +1,37 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/count.h>
+#include <thrust/device_vector.h>
+#include <thrust/remove.h>
+
+#include <cuda/memory_pool>
+#include <cuda/std/complex>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements, bit_entropy::_1_000, T{0}, T{42});
+  const auto count            = thrust::count(thrust::device, in.begin(), in.end(), T{42});
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements - count);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::remove(policy(alloc, launch), in.begin(), in.end(), T{42});
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/remove_copy/basic.cu

@@ -0,0 +1,38 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/count.h>
+#include <thrust/device_vector.h>
+#include <thrust/remove.h>
+
+#include <cuda/memory_pool>
+#include <cuda/std/complex>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements, bit_entropy::_1_000, T{0}, T{42});
+  thrust::device_vector<T> out(elements, thrust::no_init);
+  const auto count = thrust::count(thrust::device, in.begin(), in.end(), T{42});
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements - count);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(thrust::remove_copy(policy(alloc, launch), in.begin(), in.end(), out.begin(), T{42}));
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/remove_copy_if/basic.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/remove.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+struct is_even
+{
+  template <class T>
+  __device__ constexpr bool operator()(const T& val) const noexcept
+  {
+    return static_cast<int>(val) % 2 == 0;
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements, bit_entropy::_1_000, T{0}, T{42});
+  thrust::device_vector<T> out(elements, thrust::no_init);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements / 2);
+
+  caching_allocator_t alloc{};
+
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      do_not_optimize(thrust::remove_copy_if(policy(alloc, launch), in.begin(), in.end(), out.begin(), is_even{}));
+    });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/remove_if/basic.cu

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/remove.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+struct is_even
+{
+  template <class T>
+  __device__ constexpr bool operator()(const T& val) const noexcept
+  {
+    return static_cast<int>(val) % 2 == 0;
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements, bit_entropy::_1_000, T{0}, T{42});
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements / 2);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::remove_if(policy(alloc, launch), in.begin(), in.end(), is_even{});
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/replace/basic.cu

@@ -0,0 +1,34 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/replace.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements, bit_entropy::_1_000, T{0}, T{42});
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::replace(policy(alloc, launch), in.begin(), in.end(), 42, 1337);
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/replace_copy/basic.cu

@@ -0,0 +1,35 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/replace.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements, bit_entropy::_1_000, T{0}, T{42});
+  thrust::device_vector<T> out(elements, thrust::no_init);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc{};
+
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      do_not_optimize(thrust::replace_copy(policy(alloc, launch), in.begin(), in.end(), out.begin(), 42, 1337));
+    });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/replace_copy_if/basic.cu

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/replace.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+struct equal_to_42
+{
+  template <class T>
+  __device__ constexpr bool operator()(const T& val) const noexcept
+  {
+    return val == static_cast<T>(42);
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements, bit_entropy::_1_000, T{0}, T{42});
+  thrust::device_vector<T> out(elements, thrust::no_init);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc{};
+
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      do_not_optimize(
+        thrust::replace_copy_if(policy(alloc, launch), in.begin(), in.end(), out.begin(), equal_to_42{}, 1337));
+    });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/replace_if/basic.cu

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/replace.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+struct equal_to_42
+{
+  template <class T>
+  __device__ constexpr bool operator()(const T& val) const noexcept
+  {
+    return val == static_cast<T>(42);
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements, bit_entropy::_1_000, T{0}, T{42});
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::replace_if(policy(alloc, launch), in.begin(), in.end(), equal_to_42{}, 1337);
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/scan/exclusive/by_key.cu

@@ -0,0 +1,47 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/scan.h>
+
+#include "nvbench_helper.cuh"
+
+template <class KeyT, class ValueT>
+static void scan(nvbench::state& state, nvbench::type_list<KeyT, ValueT>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<ValueT> in_vals(elements);
+  thrust::device_vector<ValueT> out_vals(elements);
+  thrust::device_vector<KeyT> keys = generate.uniform.key_segments(elements, 0, 5200);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<ValueT>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::exclusive_scan_by_key(
+                 policy(alloc, launch), keys.cbegin(), keys.cend(), in_vals.cbegin(), out_vals.begin());
+             });
+}
+
+using key_types = all_types;
+using value_types =
+  nvbench::type_list<int8_t,
+                     int16_t,
+                     int32_t,
+                     int64_t
+#if _CCCL_HAS_INT128()
+                     ,
+                     int128_t
+#endif
+                     >;
+
+NVBENCH_BENCH_TYPES(scan, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/scan/exclusive/max.cu

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/scan.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      thrust::exclusive_scan(policy(alloc, launch), input.cbegin(), input.cend(), output.begin(), T{}, max_t{});
+    });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(all_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/scan/exclusive/sum.cu

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/scan.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::exclusive_scan(policy(alloc, launch), input.cbegin(), input.cend(), output.begin());
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(all_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/scan/inclusive/by_key.cu

@@ -0,0 +1,47 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/scan.h>
+
+#include "nvbench_helper.cuh"
+
+template <class KeyT, class ValueT>
+static void scan(nvbench::state& state, nvbench::type_list<KeyT, ValueT>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<ValueT> in_vals(elements);
+  thrust::device_vector<ValueT> out_vals(elements);
+  thrust::device_vector<KeyT> keys = generate.uniform.key_segments(elements, 0, 5200);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<ValueT>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::inclusive_scan_by_key(
+                 policy(alloc, launch), keys.cbegin(), keys.cend(), in_vals.cbegin(), out_vals.begin());
+             });
+}
+
+using key_types = all_types;
+using value_types =
+  nvbench::type_list<int8_t,
+                     int16_t,
+                     int32_t,
+                     int64_t
+#if _CCCL_HAS_INT128()
+                     ,
+                     int128_t
+#endif
+                     >;
+
+NVBENCH_BENCH_TYPES(scan, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/scan/inclusive/max.cu

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/scan.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::inclusive_scan(policy(alloc, launch), input.cbegin(), input.cend(), output.begin(), max_t{});
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(all_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/scan/inclusive/sum.cu

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/scan.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> input = generate(elements);
+  thrust::device_vector<T> output(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::inclusive_scan(policy(alloc, launch), input.cbegin(), input.cend(), output.begin());
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(all_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/set_operations/base.cuh

@@ -0,0 +1,54 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#pragma once
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/set_operations.h>
+#include <thrust/sort.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T, typename OpT>
+static void basic(nvbench::state& state, nvbench::type_list<T>, OpT op)
+{
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto size_ratio     = static_cast<std::size_t>(state.get_int64("SizeRatio"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  const auto elements_in_A = static_cast<std::size_t>(static_cast<double>(size_ratio * elements) / 100.0f);
+
+  thrust::device_vector<T> input = generate(elements, entropy);
+  thrust::device_vector<T> output(elements);
+
+  thrust::sort(input.begin(), input.begin() + elements_in_A);
+  thrust::sort(input.begin() + elements_in_A, input.end());
+
+  caching_allocator_t alloc;
+  // not a warm-up run, we need to run once to determine the size of the output
+  const auto result_ends =
+    op(policy(alloc),
+       input.cbegin(),
+       input.cbegin() + elements_in_A,
+       input.cbegin() + elements_in_A,
+       input.cend(),
+       output.begin());
+  const std::size_t elements_in_AB = ::cuda::std::distance(output.begin(), result_ends);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements_in_AB);
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               op(policy(alloc, launch),
+                  input.cbegin(),
+                  input.cbegin() + elements_in_A,
+                  input.cbegin() + elements_in_A,
+                  input.cend(),
+                  output.begin());
+             });
+}
+
+using types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t>;

cccl_upstream/thrust/benchmarks/bench/set_operations/by_key.cuh

@@ -0,0 +1,66 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#pragma once
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/set_operations.h>
+#include <thrust/sort.h>
+
+#include "nvbench_helper.cuh"
+
+template <class KeyT, class ValueT, class OpT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT>, OpT op)
+{
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto size_ratio     = static_cast<std::size_t>(state.get_int64("SizeRatio"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  const auto elements_in_A = static_cast<std::size_t>(static_cast<double>(size_ratio * elements) / 100.0f);
+
+  thrust::device_vector<KeyT> in_keys = generate(elements, entropy);
+  thrust::device_vector<KeyT> out_keys(elements);
+
+  thrust::device_vector<ValueT> in_vals(elements);
+  thrust::device_vector<ValueT> out_vals(elements);
+
+  thrust::sort(in_keys.begin(), in_keys.begin() + elements_in_A);
+  thrust::sort(in_keys.begin() + elements_in_A, in_keys.end());
+
+  caching_allocator_t alloc;
+  // not a warm-up run, we need to run once to determine the size of the output
+  auto result_ends = op(
+    policy(alloc),
+    in_keys.cbegin(),
+    in_keys.cbegin() + elements_in_A,
+    in_keys.cbegin() + elements_in_A,
+    in_keys.cend(),
+    in_vals.cbegin(),
+    in_vals.cbegin() + elements_in_A,
+    out_keys.begin(),
+    out_vals.begin());
+  const std::size_t elements_in_AB = ::cuda::std::distance(out_keys.begin(), result_ends.first);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_writes<KeyT>(elements_in_AB);
+  state.add_global_memory_reads<ValueT>(OpT::read_all_values ? elements : elements_in_A);
+  state.add_global_memory_writes<ValueT>(elements_in_AB);
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               op(policy(alloc, launch),
+                  in_keys.cbegin(),
+                  in_keys.cbegin() + elements_in_A,
+                  in_keys.cbegin() + elements_in_A,
+                  in_keys.cend(),
+                  in_vals.cbegin(),
+                  in_vals.cbegin() + elements_in_A,
+                  out_keys.begin(),
+                  out_vals.begin());
+             });
+}
+
+using key_types   = nvbench::type_list<int8_t, int16_t, int32_t, int64_t>;
+using value_types = nvbench::type_list<int8_t, int64_t>;

cccl_upstream/thrust/benchmarks/bench/set_operations/difference.cu

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "base.cuh"
+
+struct op_t
+{
+  template <class PolicyT, class InputIterator1, class InputIterator2, class OutputIterator>
+  __host__ OutputIterator operator()(
+    const PolicyT& policy,
+    InputIterator1 first1,
+    InputIterator1 last1,
+    InputIterator2 first2,
+    InputIterator2 last2,
+    OutputIterator result) const
+  {
+    return thrust::set_difference(policy, first1, last1, first2, last2, result);
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/difference_by_key.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "by_key.cuh"
+
+struct op_t
+{
+  static constexpr bool read_all_values = true;
+
+  template <class PolicyT,
+            class InputIterator1,
+            class InputIterator2,
+            class InputIterator3,
+            class InputIterator4,
+            class OutputIterator1,
+            class OutputIterator2>
+  __host__ cuda::std::pair<OutputIterator1, OutputIterator2> operator()(
+    const PolicyT& policy,
+    InputIterator1 keys_first1,
+    InputIterator1 keys_last1,
+    InputIterator2 keys_first2,
+    InputIterator2 keys_last2,
+    InputIterator3 values_first1,
+    InputIterator4 values_first2,
+    OutputIterator1 keys_result,
+    OutputIterator2 values_result) const
+  {
+    return thrust::set_difference_by_key(
+      policy, keys_first1, keys_last1, keys_first2, keys_last2, values_first1, values_first2, keys_result, values_result);
+  }
+};
+
+template <class KeyT, class ValueT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/intersection.cu

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "base.cuh"
+
+struct op_t
+{
+  template <class PolicyT, class InputIterator1, class InputIterator2, class OutputIterator>
+  __host__ OutputIterator operator()(
+    const PolicyT& policy,
+    InputIterator1 first1,
+    InputIterator1 last1,
+    InputIterator2 first2,
+    InputIterator2 last2,
+    OutputIterator result) const
+  {
+    return thrust::set_intersection(policy, first1, last1, first2, last2, result);
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/intersection_by_key.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "by_key.cuh"
+
+struct op_t
+{
+  static constexpr bool read_all_values = false;
+
+  template <class PolicyT,
+            class InputIterator1,
+            class InputIterator2,
+            class InputIterator3,
+            class InputIterator4,
+            class OutputIterator1,
+            class OutputIterator2>
+  __host__ cuda::std::pair<OutputIterator1, OutputIterator2> operator()(
+    const PolicyT& policy,
+    InputIterator1 keys_first1,
+    InputIterator1 keys_last1,
+    InputIterator2 keys_first2,
+    InputIterator2 keys_last2,
+    InputIterator3 values_first1,
+    InputIterator4 /* values_first2 */,
+    OutputIterator1 keys_result,
+    OutputIterator2 values_result) const
+  {
+    return thrust::set_intersection_by_key(
+      policy, keys_first1, keys_last1, keys_first2, keys_last2, values_first1, keys_result, values_result);
+  }
+};
+
+template <class KeyT, class ValueT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/symmetric_difference.cu

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "base.cuh"
+
+struct op_t
+{
+  template <class PolicyT, class InputIterator1, class InputIterator2, class OutputIterator>
+  __host__ OutputIterator operator()(
+    const PolicyT& policy,
+    InputIterator1 first1,
+    InputIterator1 last1,
+    InputIterator2 first2,
+    InputIterator2 last2,
+    OutputIterator result) const
+  {
+    return thrust::set_symmetric_difference(policy, first1, last1, first2, last2, result);
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/symmetric_difference_by_key.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "by_key.cuh"
+
+struct op_t
+{
+  static constexpr bool read_all_values = true;
+
+  template <class PolicyT,
+            class InputIterator1,
+            class InputIterator2,
+            class InputIterator3,
+            class InputIterator4,
+            class OutputIterator1,
+            class OutputIterator2>
+  __host__ cuda::std::pair<OutputIterator1, OutputIterator2> operator()(
+    const PolicyT& policy,
+    InputIterator1 keys_first1,
+    InputIterator1 keys_last1,
+    InputIterator2 keys_first2,
+    InputIterator2 keys_last2,
+    InputIterator3 values_first1,
+    InputIterator4 values_first2,
+    OutputIterator1 keys_result,
+    OutputIterator2 values_result) const
+  {
+    return thrust::set_symmetric_difference_by_key(
+      policy, keys_first1, keys_last1, keys_first2, keys_last2, values_first1, values_first2, keys_result, values_result);
+  }
+};
+
+template <class KeyT, class ValueT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/union.cu

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "base.cuh"
+
+struct op_t
+{
+  template <class PolicyT, class InputIterator1, class InputIterator2, class OutputIterator>
+  __host__ OutputIterator operator()(
+    const PolicyT& policy,
+    InputIterator1 first1,
+    InputIterator1 last1,
+    InputIterator2 first2,
+    InputIterator2 last2,
+    OutputIterator result) const
+  {
+    return thrust::set_union(policy, first1, last1, first2, last2, result);
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/union_by_key.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "by_key.cuh"
+
+struct op_t
+{
+  static constexpr bool read_all_values = true;
+
+  template <class PolicyT,
+            class InputIterator1,
+            class InputIterator2,
+            class InputIterator3,
+            class InputIterator4,
+            class OutputIterator1,
+            class OutputIterator2>
+  __host__ cuda::std::pair<OutputIterator1, OutputIterator2> operator()(
+    const PolicyT& policy,
+    InputIterator1 keys_first1,
+    InputIterator1 keys_last1,
+    InputIterator2 keys_first2,
+    InputIterator2 keys_last2,
+    InputIterator3 values_first1,
+    InputIterator4 values_first2,
+    OutputIterator1 keys_result,
+    OutputIterator2 values_result) const
+  {
+    return thrust::set_union_by_key(
+      policy, keys_first1, keys_last1, keys_first2, keys_last2, values_first1, values_first2, keys_result, values_result);
+  }
+};
+
+template <class KeyT, class ValueT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/shuffle/basic.cu

@@ -0,0 +1,72 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/random.h>
+#include <thrust/shuffle.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> data(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  auto do_engine = [&](auto&& engine_constructor) {
+    caching_allocator_t alloc;
+    state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+               [&](nvbench::launch& launch) {
+                 thrust::shuffle(policy(alloc, launch), data.begin(), data.end(), engine_constructor());
+               });
+  };
+
+  const auto rng_engine = state.get_string("Engine");
+  if (rng_engine == "minstd")
+  {
+    do_engine([] {
+      return thrust::random::minstd_rand{};
+    });
+  }
+  else if (rng_engine == "ranlux24")
+  {
+    do_engine([] {
+      return thrust::random::ranlux24{};
+    });
+  }
+  else if (rng_engine == "ranlux48")
+  {
+    do_engine([] {
+      return thrust::random::ranlux48{};
+    });
+  }
+  else if (rng_engine == "taus88")
+  {
+    do_engine([] {
+      return thrust::random::taus88{};
+    });
+  }
+}
+
+using types =
+  nvbench::type_list<int8_t,
+                     int16_t,
+                     int32_t,
+                     int64_t
+#if _CCCL_HAS_INT128()
+                     ,
+                     int128_t
+#endif
+                     >;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Engine", {"minstd", "ranlux24", "ranlux48", "taus88"});

cccl_upstream/thrust/benchmarks/bench/sort/keys.cu

@@ -0,0 +1,38 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/sort.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  thrust::device_vector<T> input = generate(elements, entropy);
+
+  thrust::device_vector<T> vec(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::timer | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch, auto& timer) {
+               vec = input;
+               timer.start();
+               thrust::sort(policy(alloc, launch), vec.begin(), vec.end());
+               timer.stop();
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"});

cccl_upstream/thrust/benchmarks/bench/sort/keys_custom.cu

@@ -0,0 +1,38 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/sort.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  thrust::device_vector<T> input = generate(elements, entropy);
+
+  thrust::device_vector<T> vec(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::timer | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch, auto& timer) {
+               vec = input;
+               timer.start();
+               thrust::sort(policy(alloc, launch), vec.begin(), vec.end(), less_t{});
+               timer.stop();
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"});

cccl_upstream/thrust/benchmarks/bench/sort/pairs.cu

@@ -0,0 +1,45 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/sort.h>
+
+#include "nvbench_helper.cuh"
+
+template <class KeyT, class ValueT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT>)
+{
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  thrust::device_vector<KeyT> in_keys = generate(elements, entropy);
+  thrust::device_vector<KeyT> keys(elements);
+
+  thrust::device_vector<ValueT> in_vals = generate(elements);
+  thrust::device_vector<ValueT> vals(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<KeyT>(elements);
+  state.add_global_memory_writes<ValueT>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::timer | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch, auto& timer) {
+               keys = in_keys;
+               vals = in_vals;
+               timer.start();
+               thrust::sort_by_key(policy(alloc, launch), keys.begin(), keys.end(), vals.begin());
+               timer.stop();
+             });
+}
+
+using key_types   = integral_types;
+using value_types = integral_types;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"});

cccl_upstream/thrust/benchmarks/bench/sort/pairs_custom.cu

@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/sort.h>
+
+#include "nvbench_helper.cuh"
+
+template <class KeyT, class ValueT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT>)
+{
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  thrust::device_vector<KeyT> in_keys = generate(elements, entropy);
+  thrust::device_vector<KeyT> keys(elements);
+
+  thrust::device_vector<ValueT> in_vals = generate(elements);
+  thrust::device_vector<ValueT> vals(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<KeyT>(elements);
+  state.add_global_memory_writes<ValueT>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::timer | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch, auto& timer) {
+               keys = in_keys;
+               vals = in_vals;
+               timer.start();
+               thrust::sort_by_key(policy(alloc, launch), keys.begin(), keys.end(), vals.begin(), less_t{});
+               timer.stop();
+             });
+}
+
+using key_types   = integral_types;
+using value_types = integral_types;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"});

cccl_upstream/thrust/benchmarks/bench/swap_ranges/basic.cu

@@ -0,0 +1,30 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include <thrust/device_vector.h>
+#include <thrust/swap.h>
+
+#include <nvbench_helper.cuh>
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements        = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> a = generate(elements);
+  thrust::device_vector<T> b = generate(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(2 * elements);
+  state.add_global_memory_writes<T>(2 * elements);
+
+  caching_allocator_t alloc; // swap_ranges shouldn't allocate, but let's be consistent
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::swap_ranges(policy(alloc, launch), a.begin(), a.end(), b.begin());
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(integral_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/tabulate/basic.cu

@@ -0,0 +1,71 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/sequence.h>
+#include <thrust/tabulate.h>
+
+#include <nvbench_helper.cuh>
+
+#include "thrust/detail/raw_pointer_cast.h"
+
+template <typename T>
+static void sequence(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> output(elements, thrust::no_init);
+
+  state.add_element_count(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               // sequence is implemented via thrust::tabulate
+               thrust::sequence(policy(alloc, launch), output.begin(), output.end());
+             });
+}
+
+NVBENCH_BENCH_TYPES(sequence, NVBENCH_TYPE_AXES(integral_types))
+  .set_name("sequence")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));
+
+template <class T>
+struct seg_size_t
+{
+  T* d_offsets{};
+
+  template <class OffsetT>
+  __device__ T operator()(OffsetT i)
+  {
+    return static_cast<T>(d_offsets[i + 1] - d_offsets[i]);
+  }
+};
+
+template <typename T>
+static void seg_size(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> input(elements + 1);
+  thrust::device_vector<T> output(elements, thrust::no_init);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements + 1);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  seg_size_t<T> op{thrust::raw_pointer_cast(input.data())};
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::tabulate(policy(alloc, launch), output.begin(), output.end(), op);
+             });
+}
+
+NVBENCH_BENCH_TYPES(seg_size, NVBENCH_TYPE_AXES(integral_types))
+  .set_name("seg_size")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/transform/babelstream.cu

@@ -0,0 +1,173 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/iterator/zip_iterator.h>
+#include <thrust/transform.h>
+#include <thrust/zip_function.h>
+
+#include <cuda/__functional/address_stability.h>
+
+#include <nvbench_helper.cuh>
+
+// The benchmarks are inspired by the BabelStream thrust version:
+// https://github.com/UoB-HPC/BabelStream/blob/main/src/thrust/ThrustStream.cu
+
+// Modified from BabelStream to also work for integers
+constexpr auto startA      = 1; // BabelStream: 0.1
+constexpr auto startB      = 2; // BabelStream: 0.2
+constexpr auto startC      = 3; // BabelStream: 0.1
+constexpr auto startScalar = 4; // BabelStream: 0.4
+
+using element_types = nvbench::type_list<std::int8_t, std::int16_t, float, double, __int128>;
+// Different benchmarks use a different number of buffers. H200/B200 can fit 2^31 elements for all benchmarks and types.
+// Upstream BabelStream uses 2^25. Allocation failure just skips the benchmark
+auto array_size_powers = std::vector<std::int64_t>{25, 31};
+
+template <typename... Args>
+void bench_transform(nvbench::state& state, Args&&... args)
+{
+  caching_allocator_t alloc; // transform shouldn't allocate, but let's be consistent
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::transform(policy(alloc, launch), ::cuda::std::forward<Args>(args)...);
+             });
+}
+
+template <typename T>
+static void mul(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> b(n, startB);
+  thrust::device_vector<T> c(n, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(n);
+
+  const T scalar = startScalar;
+  bench_transform(
+    state, c.begin(), c.end(), b.begin(), cuda::proclaim_copyable_arguments([=] _CCCL_DEVICE(const T& ci) {
+      return ci * scalar;
+    }));
+}
+
+NVBENCH_BENCH_TYPES(mul, NVBENCH_TYPE_AXES(element_types))
+  .set_name("mul")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", array_size_powers);
+
+template <typename T>
+static void add(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> a(n, startA);
+  thrust::device_vector<T> b(n, startB);
+  thrust::device_vector<T> c(n, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+
+  bench_transform(
+    state,
+    a.begin(),
+    a.end(),
+    b.begin(),
+    c.begin(),
+    cuda::proclaim_copyable_arguments([] _CCCL_DEVICE(const T& ai, const T& bi) -> T {
+      return ai + bi;
+    }));
+}
+
+NVBENCH_BENCH_TYPES(add, NVBENCH_TYPE_AXES(element_types))
+  .set_name("add")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", array_size_powers);
+
+template <typename T>
+static void triad(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> a(n, startA);
+  thrust::device_vector<T> b(n, startB);
+  thrust::device_vector<T> c(n, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+
+  const T scalar = startScalar;
+  bench_transform(
+    state,
+    b.begin(),
+    b.end(),
+    c.begin(),
+    a.begin(),
+    cuda::proclaim_copyable_arguments([=] _CCCL_DEVICE(const T& bi, const T& ci) {
+      return bi + scalar * ci;
+    }));
+}
+
+NVBENCH_BENCH_TYPES(triad, NVBENCH_TYPE_AXES(element_types))
+  .set_name("triad")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", array_size_powers);
+
+template <typename T>
+static void nstream(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> a(n, startA);
+  thrust::device_vector<T> b(n, startB);
+  thrust::device_vector<T> c(n, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(3 * n);
+  state.add_global_memory_writes<T>(n);
+
+  const T scalar = startScalar;
+  bench_transform(
+    state,
+    thrust::make_zip_iterator(a.begin(), b.begin(), c.begin()),
+    thrust::make_zip_iterator(a.end(), b.end(), c.end()),
+    a.begin(),
+    thrust::make_zip_function(cuda::proclaim_copyable_arguments([=] _CCCL_DEVICE(const T& ai, const T& bi, const T& ci) {
+      return ai + bi + scalar * ci;
+    })));
+}
+
+NVBENCH_BENCH_TYPES(nstream, NVBENCH_TYPE_AXES(element_types))
+  .set_name("nstream")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", array_size_powers);
+
+// variation of nstream requiring a stable parameter address because it recovers the element index
+template <typename T>
+static void nstream_stable(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> a(n, startA);
+  thrust::device_vector<T> b(n, startB);
+  thrust::device_vector<T> c(n, startC);
+
+  const T* a_start = thrust::raw_pointer_cast(a.data());
+  const T* b_start = thrust::raw_pointer_cast(b.data());
+  const T* c_start = thrust::raw_pointer_cast(c.data());
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(3 * n);
+  state.add_global_memory_writes<T>(n);
+
+  const T scalar = startScalar;
+  bench_transform(state, a.begin(), a.end(), a.begin(), [=] _CCCL_DEVICE(const T& ai) {
+    const auto i = &ai - a_start;
+    return ai + b_start[i] + scalar * c_start[i];
+  });
+}
+
+NVBENCH_BENCH_TYPES(nstream_stable, NVBENCH_TYPE_AXES(element_types))
+  .set_name("nstream_stable")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", array_size_powers);

cccl_upstream/thrust/benchmarks/bench/transform/fib.cu

@@ -0,0 +1,62 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/transform.h>
+
+#include <nvbench_helper.cuh>
+
+template <class InT, class OutT>
+struct fib_t
+{
+  __device__ OutT operator()(InT n)
+  {
+    OutT t1 = 0;
+    OutT t2 = 1;
+
+    if (n <= 1)
+    {
+      return t1;
+    }
+    else if (n == 2)
+    {
+      return t2;
+    }
+    for (InT i = 3; i <= n; ++i)
+    {
+      const auto next = t1 + t2;
+      t1              = t2;
+      t2              = next;
+    }
+
+    return t2;
+  }
+};
+
+template <typename T>
+static void fib(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> input = generate(elements, bit_entropy::_1_000, T{0}, T{42});
+  thrust::device_vector<T> output(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<nvbench::uint32_t>(elements);
+
+  fib_t<T, nvbench::uint32_t> op{};
+  caching_allocator_t alloc; // transform shouldn't allocate, but let's be consistent
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::transform(policy(alloc, launch), input.cbegin(), input.cend(), output.begin(), op);
+             });
+}
+
+using types = nvbench::type_list<nvbench::uint32_t, nvbench::uint64_t>;
+
+NVBENCH_BENCH_TYPES(fib, NVBENCH_TYPE_AXES(types))
+  .set_name("fib")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/transform/if.cu

@@ -0,0 +1,38 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include <thrust/count.h>
+#include <thrust/device_vector.h>
+#include <thrust/transform.h>
+
+#include <nvbench_helper.cuh>
+
+template <typename T>
+static void negate_if(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto entropy = str_to_entropy(state.get_string("Entropy"));
+  const auto val     = lerp_min_max<T>(entropy_to_probability(entropy));
+  auto transform_op  = ::cuda::std::negate<T>{};
+  auto select_op     = less_then_t<T>{val};
+
+  thrust::device_vector<T> input = generate(n);
+  thrust::device_vector<T> output(n, thrust::no_init);
+
+  const auto selected_elements = thrust::count_if(input.cbegin(), input.cend(), select_op);
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(selected_elements);
+
+  caching_allocator_t alloc; // transform_if shouldn't allocate, but let's be consistent
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      thrust::transform_if(policy(alloc, launch), input.begin(), input.end(), output.begin(), transform_op, select_op);
+    });
+}
+
+NVBENCH_BENCH_TYPES(negate_if, NVBENCH_TYPE_AXES(integral_types))
+  //  .set_name("negate_if")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.000"});

cccl_upstream/thrust/benchmarks/bench/transform/if_stencil.cu

@@ -0,0 +1,39 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/count.h>
+#include <thrust/device_vector.h>
+#include <thrust/transform.h>
+
+#include <nvbench_helper.cuh>
+
+template <typename T>
+static void negate_if_stencil(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto entropy = str_to_entropy(state.get_string("Entropy"));
+  const auto val     = lerp_min_max<T>(entropy_to_probability(entropy));
+  auto transform_op  = ::cuda::std::negate<T>{};
+  auto select_op     = less_then_t<T>{val};
+
+  thrust::device_vector<T> input   = generate(n);
+  thrust::device_vector<T> stencil = generate(n);
+  thrust::device_vector<T> output(n, thrust::no_init);
+
+  const auto selected_elements = thrust::count_if(input.cbegin(), input.cend(), select_op);
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n + selected_elements);
+  state.add_global_memory_writes<T>(selected_elements);
+
+  caching_allocator_t alloc; // transform_if shouldn't allocate, but let's be consistent
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      thrust::transform_if(
+        policy(alloc, launch), input.begin(), input.end(), stencil.begin(), output.begin(), transform_op, select_op);
+    });
+}
+
+NVBENCH_BENCH_TYPES(negate_if_stencil, NVBENCH_TYPE_AXES(integral_types))
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.000"});

cccl_upstream/thrust/benchmarks/bench/transform/zip_transform.cu

@@ -0,0 +1,55 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+
+#include <cuda/functional>
+#include <cuda/iterator>
+
+#include <nvbench_helper.cuh>
+
+// This benchmark is intended to be compared to the nstream benchmark from babelstream.cu, so we can:
+// * detect regressions in the unpacking of a zip_transform_iterator
+
+// same variables as in basic.cu so we can compare results
+constexpr auto startA      = 1; // BabelStream: 0.1
+constexpr auto startB      = 2; // BabelStream: 0.2
+constexpr auto startC      = 3; // BabelStream: 0.1
+constexpr auto startScalar = 4; // BabelStream: 0.4
+using element_types        = nvbench::type_list<std::int8_t, std::int16_t, float, double, __int128>;
+auto array_size_powers     = std::vector<std::int64_t>{25, 31};
+
+template <typename T>
+static void nstream_zip_transform(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> a(n, startA);
+  thrust::device_vector<T> b(n, startB);
+  thrust::device_vector<T> c(n, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(3 * n);
+  state.add_global_memory_writes<T>(n);
+
+  const T scalar = startScalar;
+  auto lambda    = cuda::proclaim_copyable_arguments([scalar] _CCCL_DEVICE(const T& ai, const T& bi, const T& ci) -> T {
+    // Needed to silence clangs -Wunused-lambda-capture. We cannot just remove it because other
+    // implementations (e.g. MSVC) will emit errors if we don't capture it. See discussion in
+    // https://reviews.llvm.org/D28467.
+    static_cast<void>(scalar);
+    return ai + bi + scalar * ci;
+  });
+  cuda::zip_transform_iterator begin{lambda, a.begin(), b.begin(), c.begin()};
+  cuda::zip_transform_iterator end{lambda, a.end(), b.end(), c.end()};
+  caching_allocator_t alloc; // transform shouldn't allocate, but let's be consistent
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::copy(policy(alloc, launch), begin, end, a.begin());
+             });
+}
+
+NVBENCH_BENCH_TYPES(nstream_zip_transform, NVBENCH_TYPE_AXES(element_types))
+  .set_name("nstream")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", array_size_powers);

cccl_upstream/thrust/benchmarks/bench/transform_exclusive_scan/basic.cu

@@ -0,0 +1,52 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <thrust/device_vector.h>
+#include <thrust/transform_scan.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <class T>
+struct times_two
+{
+  _CCCL_DEVICE constexpr T operator()(const T val) const noexcept
+  {
+    return 2 * val;
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<T> out(elements, thrust::no_init);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc{};
+
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      do_not_optimize(thrust::transform_exclusive_scan(
+        policy(alloc, launch), in.begin(), in.end(), out.begin(), times_two<T>{}, T{42}, cuda::std::plus<T>{}));
+    });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/transform_inclusive_scan/basic.cu

@@ -0,0 +1,78 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <thrust/device_vector.h>
+#include <thrust/transform_scan.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <class T>
+struct times_two
+{
+  _CCCL_DEVICE constexpr T operator()(const T val) const noexcept
+  {
+    return 2 * val;
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<T> out(elements, thrust::no_init);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(thrust::transform_inclusive_scan(
+                 policy(alloc, launch), in.begin(), in.end(), out.begin(), times_two<T>{}, cuda::std::plus<T>{}));
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("basic")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));
+
+template <typename T>
+static void with_init(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<T> out(elements, thrust::no_init);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc{};
+
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      do_not_optimize(thrust::transform_inclusive_scan(
+        policy(alloc, launch), in.begin(), in.end(), out.begin(), times_two<T>{}, T{42}, cuda::std::plus<T>{}));
+    });
+}
+
+NVBENCH_BENCH_TYPES(with_init, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("with_init")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/transform_reduce/sum.cu

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/transform_reduce.h>
+
+#include "nvbench_helper.cuh"
+
+template <class T>
+struct square_t
+{
+  __host__ __device__ T operator()(const T& x) const
+  {
+    return x * x;
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               do_not_optimize(thrust::transform_reduce(
+                 policy(alloc, launch), in.begin(), in.end(), square_t<T>{}, T{}, ::cuda::std::plus<T>{}));
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/transform_reduce/unary.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/transform_reduce.h>
+
+#include <cuda/memory_pool>
+#include <cuda/stream>
+
+#include "nvbench_helper.cuh"
+
+template <class T>
+struct plus_one
+{
+  template <class U>
+  [[nodiscard]] __device__ constexpr T operator()(const U val) const noexcept
+  {
+    return static_cast<T>(val + 1);
+  }
+};
+
+template <typename T>
+static void unary(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> in = generate(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(1);
+
+  caching_allocator_t alloc{};
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+    do_not_optimize(
+      thrust::transform_reduce(policy(alloc, launch), in.begin(), in.end(), plus_one<T>{}, 42, cuda::std::plus<T>{}));
+  });
+}
+
+NVBENCH_BENCH_TYPES(unary, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/uninitialized_copy/basic.cu

@@ -0,0 +1,66 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include <thrust/device_vector.h>
+#include <thrust/uninitialized_copy.h>
+
+#include <nvbench_helper.cuh>
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> input(elements, T{0xAA});
+  thrust::device_vector<T> output(elements, thrust::default_init);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::uninitialized_copy(policy(alloc, launch), input.cbegin(), input.cend(), output.begin());
+             });
+}
+
+// Not allowed to be copied using std::memcpy or cudaMemcpy. Cannot use TMA copies.
+struct no_copy
+{
+  nvbench::uint32_t a;
+
+  no_copy() = default;
+
+  _CCCL_HOST_DEVICE no_copy(nvbench::uint32_t i)
+      : a(i)
+  {}
+
+  // the user-defined copy constructor prevents the type from being trivially copyable
+  // NOLINTNEXTLINE(modernize-use-equals-default)
+  _CCCL_HOST_DEVICE no_copy(const no_copy& nt)
+      : a(nt.a)
+  {}
+};
+
+static_assert(::cuda::std::is_trivially_default_constructible_v<no_copy>);
+static_assert(!::cuda::std::is_trivially_copyable_v<no_copy>); // as required by the C++ standard
+static_assert(!thrust::is_trivially_relocatable_v<no_copy>); // thrust uses this check internally
+
+// Requires use of placement new
+struct no_construct
+{
+  nvbench::uint32_t a = 1337;
+};
+
+static_assert(!::cuda::std::is_trivially_default_constructible_v<no_construct>);
+static_assert(::cuda::std::is_trivially_copyable_v<no_construct>); // as required by the C++ standard
+static_assert(thrust::is_trivially_relocatable_v<no_construct>); // thrust uses this check internally
+
+using types =
+  nvbench::type_list<nvbench::uint8_t, nvbench::uint16_t, nvbench::uint32_t, nvbench::uint64_t, no_copy, no_construct>;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/unique/basic.cu

@@ -0,0 +1,40 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/unique.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  const std::size_t min_segment_size = 1;
+  const std::size_t max_segment_size = static_cast<std::size_t>(state.get_int64("MaxSegSize"));
+
+  thrust::device_vector<T> input = generate.uniform.key_segments(elements, min_segment_size, max_segment_size);
+  thrust::device_vector<T> output(elements);
+
+  caching_allocator_t alloc;
+  // not a warm-up run, we need to run once to determine the size of the output
+  const auto new_end             = thrust::unique_copy(policy(alloc), input.cbegin(), input.cend(), output.begin());
+  const std::size_t unique_items = ::cuda::std::distance(output.begin(), new_end);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(unique_items);
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::unique_copy(policy(alloc, launch), input.cbegin(), input.cend(), output.begin());
+             });
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(fundamental_types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegSize", {1, 4, 8});

cccl_upstream/thrust/benchmarks/bench/unique/by_key.cu

@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/unique.h>
+
+#include "nvbench_helper.cuh"
+
+template <class KeyT, class ValueT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  const std::size_t min_segment_size = 1;
+  const std::size_t max_segment_size = static_cast<std::size_t>(state.get_int64("MaxSegSize"));
+
+  thrust::device_vector<KeyT> in_keys = generate.uniform.key_segments(elements, min_segment_size, max_segment_size);
+  thrust::device_vector<KeyT> out_keys(elements);
+  thrust::device_vector<ValueT> in_vals(elements);
+  thrust::device_vector<ValueT> out_vals(elements);
+
+  caching_allocator_t alloc;
+  // not a warm-up run, we need to run once to determine the size of the output
+  const auto [new_key_end, new_val_end] = thrust::unique_by_key_copy(
+    policy(alloc), in_keys.cbegin(), in_keys.cend(), in_vals.cbegin(), out_keys.begin(), out_vals.begin());
+
+  const std::size_t unique_elements = ::cuda::std::distance(out_keys.begin(), new_key_end);
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_writes<KeyT>(unique_elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<ValueT>(unique_elements);
+
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      thrust::unique_by_key_copy(
+        policy(alloc, launch), in_keys.cbegin(), in_keys.cend(), in_vals.cbegin(), out_keys.begin(), out_vals.begin());
+    });
+}
+
+using key_types =
+  nvbench::type_list<int8_t,
+                     int16_t,
+                     int32_t,
+                     int64_t
+#if _CCCL_HAS_INT128()
+                     ,
+                     int128_t
+#endif
+                     >;
+using value_types = all_types;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegSize", {1, 8});

cccl_upstream/thrust/benchmarks/bench/vectorized_search/base.cu

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/binary_search.h>
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/sort.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements      = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto needles_ratio = static_cast<std::size_t>(state.get_int64("NeedlesRatio"));
+  const auto needles       = needles_ratio * static_cast<std::size_t>(static_cast<double>(elements) / 100.0);
+
+  thrust::device_vector<T> data = generate(elements + needles);
+  thrust::device_vector<bool> result(needles);
+  thrust::sort(data.begin(), data.begin() + elements);
+
+  state.add_element_count(needles);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::binary_search(
+                 policy(alloc, launch),
+                 data.begin(),
+                 data.begin() + elements,
+                 data.begin() + elements,
+                 data.end(),
+                 result.begin());
+             });
+}
+
+using types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t>;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_int64_axis("NeedlesRatio", {1, 25, 50});

cccl_upstream/thrust/benchmarks/bench/vectorized_search/lower_bound.cu

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/binary_search.h>
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/sort.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements      = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto needles_ratio = static_cast<std::size_t>(state.get_int64("NeedlesRatio"));
+  const auto needles       = needles_ratio * static_cast<std::size_t>(static_cast<double>(elements) / 100.0);
+
+  thrust::device_vector<T> data = generate(elements + needles);
+  thrust::device_vector<T> result(needles);
+  thrust::sort(data.begin(), data.begin() + elements);
+
+  state.add_element_count(needles);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::lower_bound(
+                 policy(alloc, launch),
+                 data.begin(),
+                 data.begin() + elements,
+                 data.begin() + elements,
+                 data.end(),
+                 result.begin());
+             });
+}
+
+using types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t>;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_int64_axis("NeedlesRatio", {1, 25, 50});

cccl_upstream/thrust/benchmarks/bench/vectorized_search/upper_bound.cu

@@ -0,0 +1,43 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/binary_search.h>
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/sort.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements      = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto needles_ratio = static_cast<std::size_t>(state.get_int64("NeedlesRatio"));
+  const auto needles       = needles_ratio * static_cast<std::size_t>(static_cast<double>(elements) / 100.0);
+
+  thrust::device_vector<T> data = generate(elements + needles);
+  thrust::device_vector<T> result(needles);
+  thrust::sort(data.begin(), data.begin() + elements);
+
+  state.add_element_count(needles);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::upper_bound(
+                 policy(alloc, launch),
+                 data.begin(),
+                 data.begin() + elements,
+                 data.begin() + elements,
+                 data.end(),
+                 result.begin());
+             });
+}
+
+using types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t>;
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_int64_axis("NeedlesRatio", {1, 25, 50});