[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/thrust/testing/cuda/CMakeLists.txt

@@ -0,0 +1,30 @@
+file(
+  GLOB test_srcs
+  RELATIVE "${CMAKE_CURRENT_LIST_DIR}"
+  CONFIGURE_DEPENDS
+  *.cu
+  *.cpp
+)
+
+foreach (thrust_target IN LISTS THRUST_TARGETS)
+  thrust_get_target_property(config_device ${thrust_target} DEVICE)
+  if (NOT config_device STREQUAL "CUDA")
+    continue()
+  endif()
+
+  foreach (test_src IN LISTS test_srcs)
+    get_filename_component(test_name "${test_src}" NAME_WLE)
+    string(PREPEND test_name "cuda.")
+
+    # Create two targets, one with RDC enabled, the other without. This tests
+    # both device-side behaviors -- the CDP kernel launch with RDC, and the
+    # serial fallback path without RDC.
+    thrust_add_test(seq_test_target ${test_name}.cdp_0 "${test_src}" ${thrust_target})
+    thrust_configure_cuda_target(${seq_test_target} RDC OFF)
+
+    if (THRUST_ENABLE_RDC_TESTS)
+      thrust_add_test(cdp_test_target ${test_name}.cdp_1 "${test_src}" ${thrust_target})
+      thrust_configure_cuda_target(${cdp_test_target} RDC ON)
+    endif()
+  endforeach()
+endforeach()

cccl_upstream/thrust/testing/cuda/adjacent_difference.cu

@@ -0,0 +1,162 @@
+#include <thrust/adjacent_difference.h>
+#include <thrust/device_free.h>
+#include <thrust/device_malloc.h>
+#include <thrust/execution_policy.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
+__global__ void adjacent_difference_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result)
+{
+  thrust::adjacent_difference(exec, first, last, result);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename BinaryFunction>
+__global__ void
+adjacent_difference_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result, BinaryFunction f)
+{
+  thrust::adjacent_difference(exec, first, last, result, f);
+}
+
+template <typename T, typename ExecutionPolicy>
+void TestAdjacentDifferenceDevice(ExecutionPolicy exec, const size_t n)
+{
+  thrust::host_vector<T> h_input   = unittest::random_samples<T>(n);
+  thrust::device_vector<T> d_input = h_input;
+
+  thrust::host_vector<T> h_output(n);
+  thrust::device_vector<T> d_output(n);
+
+  thrust::adjacent_difference(h_input.begin(), h_input.end(), h_output.begin());
+  adjacent_difference_kernel<<<1, 1>>>(exec, d_input.begin(), d_input.end(), d_output.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_output, d_output);
+
+  thrust::adjacent_difference(h_input.begin(), h_input.end(), h_output.begin(), ::cuda::std::plus<T>());
+  adjacent_difference_kernel<<<1, 1>>>(exec, d_input.begin(), d_input.end(), d_output.begin(), ::cuda::std::plus<T>());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_output, d_output);
+
+  // in-place operation
+  thrust::adjacent_difference(h_input.begin(), h_input.end(), h_input.begin(), ::cuda::std::plus<T>());
+  adjacent_difference_kernel<<<1, 1>>>(exec, d_input.begin(), d_input.end(), d_input.begin(), ::cuda::std::plus<T>());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_input, h_output); // computed previously
+  ASSERT_EQUAL(d_input, d_output); // computed previously
+}
+
+template <typename T>
+void TestAdjacentDifferenceDeviceSeq(const size_t n)
+{
+  TestAdjacentDifferenceDevice<T>(thrust::seq, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestAdjacentDifferenceDeviceSeq);
+
+template <typename T>
+void TestAdjacentDifferenceDeviceDevice(const size_t n)
+{
+  TestAdjacentDifferenceDevice<T>(thrust::device, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestAdjacentDifferenceDeviceDevice);
+#endif
+
+void TestAdjacentDifferenceCudaStreams()
+{
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::device_vector<int> input{1, 4, 6};
+  thrust::device_vector<int> output(input.size());
+
+  thrust::adjacent_difference(thrust::cuda::par.on(s), input.begin(), input.end(), output.begin());
+
+  cudaStreamSynchronize(s);
+
+  thrust::device_vector<int> ref{1, 3, 2};
+
+  ASSERT_EQUAL(output, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestAdjacentDifferenceCudaStreams);
+
+struct detect_wrong_difference
+{
+  using difference_type   = void;
+  using value_type        = long long;
+  using pointer           = void;
+  using reference         = detect_wrong_difference;
+  using iterator_category = ::cuda::std::output_iterator_tag;
+
+  bool* flag;
+
+  _CCCL_HOST_DEVICE detect_wrong_difference operator++() const
+  {
+    return *this;
+  }
+  _CCCL_HOST_DEVICE detect_wrong_difference operator*() const
+  {
+    return *this;
+  }
+  template <typename Difference>
+  _CCCL_HOST_DEVICE detect_wrong_difference operator+(Difference) const
+  {
+    return *this;
+  }
+  template <typename Index>
+  _CCCL_HOST_DEVICE detect_wrong_difference operator[](Index) const
+  {
+    return *this;
+  }
+
+  _CCCL_DEVICE void operator=(long long difference) const
+  {
+    if (difference != 1)
+    {
+      *flag = false;
+    }
+  }
+};
+
+void TestAdjacentDifferenceWithBigIndexesHelper(int magnitude)
+{
+  thrust::counting_iterator<long long> begin(1);
+  thrust::counting_iterator<long long> end = begin + (1ll << magnitude);
+  ASSERT_EQUAL(::cuda::std::distance(begin, end), 1ll << magnitude);
+
+  thrust::device_ptr<bool> all_differences_correct = thrust::device_malloc<bool>(1);
+  *all_differences_correct                         = true;
+
+  detect_wrong_difference out = {thrust::raw_pointer_cast(all_differences_correct)};
+
+  thrust::adjacent_difference(thrust::device, begin, end, out);
+
+  bool all_differences_correct_h = *all_differences_correct;
+  thrust::device_free(all_differences_correct);
+
+  ASSERT_EQUAL(all_differences_correct_h, true);
+}
+
+void TestAdjacentDifferenceWithBigIndexes()
+{
+  TestAdjacentDifferenceWithBigIndexesHelper(30);
+#ifndef THRUST_FORCE_32_BIT_OFFSET_TYPE
+  TestAdjacentDifferenceWithBigIndexesHelper(31);
+  TestAdjacentDifferenceWithBigIndexesHelper(32);
+  TestAdjacentDifferenceWithBigIndexesHelper(33);
+#endif
+}
+DECLARE_UNITTEST(TestAdjacentDifferenceWithBigIndexes);

cccl_upstream/thrust/testing/cuda/binary_search.cu

@@ -0,0 +1,24 @@
+#include <thrust/binary_search.h>
+#include <thrust/device_vector.h>
+#include <thrust/distance.h>
+#include <thrust/sequence.h>
+
+#include <cuda/std/utility>
+
+#include <unittest/unittest.h>
+
+void TestEqualRangeOnStream()
+{ // Regression test for GH issue #921 (nvbug 2173437)
+  using vector_t   = typename thrust::device_vector<int>;
+  using iterator_t = typename vector_t::iterator;
+  using result_t   = cuda::std::pair<iterator_t, iterator_t>;
+
+  vector_t input(10);
+  thrust::sequence(thrust::device, input.begin(), input.end(), 0);
+  cudaStream_t stream = nullptr;
+  result_t result     = thrust::equal_range(thrust::cuda::par.on(stream), input.begin(), input.end(), 5);
+
+  ASSERT_EQUAL(5, ::cuda::std::distance(input.begin(), result.first));
+  ASSERT_EQUAL(6, ::cuda::std::distance(input.begin(), result.second));
+}
+DECLARE_UNITTEST(TestEqualRangeOnStream);

cccl_upstream/thrust/testing/cuda/cccl_runtime_test_helper.cuh

@@ -0,0 +1,119 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+#include <cuda/__cccl_config>
+#include <cuda/algorithm>
+#include <cuda/buffer>
+#include <cuda/devices>
+#include <cuda/launch>
+#include <cuda/memory_resource>
+#include <cuda/std/cstddef>
+#include <cuda/std/initializer_list>
+#include <cuda/std/span>
+#include <cuda/stream>
+
+#include <cstdio>
+#include <vector>
+
+#include <cuda_runtime_api.h>
+
+#include <unittest/unittest.h>
+
+namespace test_runtime
+{
+[[nodiscard]] _CCCL_HOST_API inline cuda::device_ref current_test_device()
+{
+  int device = 0;
+  ASSERT_EQUAL(cudaSuccess, cudaGetDevice(&device));
+  return cuda::device_ref{device};
+}
+
+[[nodiscard]] _CCCL_HOST_API inline auto single_thread_config()
+{
+  return cuda::make_config(cuda::make_hierarchy(cuda::grid_dims(1), cuda::block_dims<1>()));
+}
+
+_CCCL_DEVICE_API inline void assert_device(bool condition, const char* expression, const char* file, int line) noexcept
+{
+  if (!condition)
+  {
+    printf("Device assertion failed: %s (%s:%d)\n", expression, file, line);
+    __trap();
+  }
+}
+
+template <typename T>
+[[nodiscard]] _CCCL_HOST_API inline auto make_host_buffer(cuda::stream_ref stream, cuda::std::size_t size)
+{
+  using resource_t = cuda::mr::synchronous_resource_adapter<cuda::mr::legacy_pinned_memory_resource>;
+  resource_t resource{cuda::mr::legacy_pinned_memory_resource{cuda::device_ref{0}}};
+  return cuda::make_buffer<T>(stream, resource, size, cuda::no_init);
+}
+
+template <typename T, typename RandomT = T>
+[[nodiscard]] _CCCL_HOST_API inline auto
+random_integers_buffer(cuda::stream_ref stream, cuda::std::size_t size, cuda::std::size_t first = 0)
+{
+  auto result = make_host_buffer<T>(stream, size);
+  stream.sync();
+
+  const auto generator = unittest::generate_random_integer<RandomT>{};
+  for (cuda::std::size_t i = 0; i < size; ++i)
+  {
+    result[i] = static_cast<T>(generator(static_cast<unsigned int>(first + i)));
+  }
+
+  return result;
+}
+
+template <typename T, typename RandomT = T>
+[[nodiscard]] _CCCL_HOST_API inline auto
+random_samples_buffer(cuda::stream_ref stream, cuda::std::size_t size, cuda::std::size_t first = 0)
+{
+  auto result = make_host_buffer<T>(stream, size);
+  stream.sync();
+
+  const auto generator = unittest::generate_random_sample<RandomT>{};
+  for (cuda::std::size_t i = 0; i < size; ++i)
+  {
+    result[i] = static_cast<T>(generator(static_cast<unsigned int>(first + i)));
+  }
+
+  return result;
+}
+
+template <typename Buffer>
+_CCCL_HOST_API inline void
+assert_equal(cuda::stream_ref stream, Buffer& buffer, cuda::std::initializer_list<int> expected)
+{
+  std::vector<int> actual(buffer.size());
+  cuda::copy_bytes(stream, buffer, actual);
+  stream.sync();
+
+  ASSERT_EQUAL(actual.size(), expected.size());
+
+  for (cuda::std::size_t i = 0; i < expected.size(); ++i)
+  {
+    ASSERT_EQUAL(expected.begin()[i], actual[i]);
+  }
+}
+
+template <typename Buffer, typename Expected>
+_CCCL_HOST_API inline void assert_equal(cuda::stream_ref stream, Buffer& buffer, const Expected& expected)
+{
+  std::vector<int> actual(buffer.size());
+  cuda::copy_bytes(stream, buffer, actual);
+  stream.sync();
+
+  ASSERT_EQUAL(actual.size(), expected.size());
+
+  for (cuda::std::size_t i = 0; i < expected.size(); ++i)
+  {
+    ASSERT_EQUAL(expected[i], actual[i]);
+  }
+}
+} // namespace test_runtime
+
+#define TEST_ASSERT_DEVICE(condition) ::test_runtime::assert_device((condition), #condition, __FILE__, __LINE__)

cccl_upstream/thrust/testing/cuda/complex.cu

@@ -0,0 +1,24 @@
+#include <cuda_fp16.h>
+
+#include <thrust/complex.h>
+#include <thrust/detail/alignment.h>
+#include <thrust/detail/preprocessor.h>
+
+#include <unittest/unittest.h>
+
+template <typename T, typename VectorT>
+void TestComplexAlignment()
+{
+  static_assert(sizeof(thrust::complex<T>) == sizeof(VectorT));
+  static_assert(alignof(thrust::complex<T>) == alignof(VectorT));
+
+  static_assert(sizeof(thrust::complex<T const>) == sizeof(VectorT));
+  static_assert(alignof(thrust::complex<T const>) == alignof(VectorT));
+}
+DECLARE_UNITTEST_WITH_NAME(THRUST_PP_EXPAND_ARGS(TestComplexAlignment<char, char2>), TestComplexCharAlignment);
+DECLARE_UNITTEST_WITH_NAME(THRUST_PP_EXPAND_ARGS(TestComplexAlignment<short, short2>), TestComplexShortAlignment);
+DECLARE_UNITTEST_WITH_NAME(THRUST_PP_EXPAND_ARGS(TestComplexAlignment<int, int2>), TestComplexIntAlignment);
+DECLARE_UNITTEST_WITH_NAME(THRUST_PP_EXPAND_ARGS(TestComplexAlignment<long, long2>), TestComplexLongAlignment);
+DECLARE_UNITTEST_WITH_NAME(THRUST_PP_EXPAND_ARGS(TestComplexAlignment<__half, __half2>), TestComplexHalfAlignment);
+DECLARE_UNITTEST_WITH_NAME(THRUST_PP_EXPAND_ARGS(TestComplexAlignment<float, float2>), TestComplexFloatAlignment);
+DECLARE_UNITTEST_WITH_NAME(THRUST_PP_EXPAND_ARGS(TestComplexAlignment<double, double2>), TestComplexDoubleAlignment);

cccl_upstream/thrust/testing/cuda/copy.cu

@@ -0,0 +1,84 @@
+#include <thrust/copy.h>
+#include <thrust/execution_policy.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
+__global__ void copy_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result)
+{
+  thrust::copy(exec, first, last, result);
+}
+
+template <typename T, typename ExecutionPolicy>
+void TestCopyDevice(ExecutionPolicy exec, size_t n)
+{
+  thrust::host_vector<T> h_src = unittest::random_integers<T>(n);
+  thrust::host_vector<T> h_dst(n);
+
+  thrust::device_vector<T> d_src = h_src;
+  thrust::device_vector<T> d_dst(n);
+
+  thrust::copy(h_src.begin(), h_src.end(), h_dst.begin());
+  copy_kernel<<<1, 1>>>(exec, d_src.begin(), d_src.end(), d_dst.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_dst, d_dst);
+}
+
+template <typename T>
+void TestCopyDeviceSeq(size_t n)
+{
+  TestCopyDevice<T>(thrust::seq, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestCopyDeviceSeq);
+
+template <typename T>
+void TestCopyDeviceDevice(size_t n)
+{
+  TestCopyDevice<T>(thrust::device, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestCopyDeviceDevice);
+
+template <typename ExecutionPolicy, typename Iterator1, typename Size, typename Iterator2>
+__global__ void copy_n_kernel(ExecutionPolicy exec, Iterator1 first, Size n, Iterator2 result)
+{
+  thrust::copy_n(exec, first, n, result);
+}
+
+template <typename T, typename ExecutionPolicy>
+void TestCopyNDevice(ExecutionPolicy exec, size_t n)
+{
+  thrust::host_vector<T> h_src = unittest::random_integers<T>(n);
+  thrust::host_vector<T> h_dst(n);
+
+  thrust::device_vector<T> d_src = h_src;
+  thrust::device_vector<T> d_dst(n);
+
+  thrust::copy_n(h_src.begin(), h_src.size(), h_dst.begin());
+  copy_n_kernel<<<1, 1>>>(exec, d_src.begin(), d_src.size(), d_dst.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_dst, d_dst);
+}
+
+template <typename T>
+void TestCopyNDeviceSeq(size_t n)
+{
+  TestCopyNDevice<T>(thrust::seq, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestCopyNDeviceSeq);
+
+template <typename T>
+void TestCopyNDeviceDevice(size_t n)
+{
+  TestCopyNDevice<T>(thrust::device, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestCopyNDeviceDevice);
+#endif

cccl_upstream/thrust/testing/cuda/copy_if.cu

@@ -0,0 +1,363 @@
+#include <thrust/copy.h>
+#include <thrust/execution_policy.h>
+#include <thrust/sequence.h>
+
+#include "thrust/iterator/transform_iterator.h"
+#include <unittest/unittest.h>
+
+template <typename T>
+struct is_even
+{
+  _CCCL_HOST_DEVICE bool operator()(T x)
+  {
+    return (static_cast<unsigned int>(x) & 1) == 0;
+  }
+};
+
+template <typename T>
+struct mod_3
+{
+  _CCCL_HOST_DEVICE unsigned int operator()(T x)
+  {
+    return static_cast<unsigned int>(x) % 3;
+  }
+};
+
+template <typename T>
+struct mod_n
+{
+  T mod;
+  _CCCL_HOST_DEVICE bool operator()(T x)
+  {
+    return (x % mod == 0) ? true : false;
+  }
+};
+
+template <typename T>
+struct multiply_n
+{
+  T multiplier;
+  _CCCL_HOST_DEVICE T operator()(T x)
+  {
+    return x * multiplier;
+  }
+};
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Predicate, typename Iterator3>
+__global__ void copy_if_kernel(
+  ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result1, Predicate pred, Iterator3 result2)
+{
+  *result2 = thrust::copy_if(exec, first, last, result1, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestCopyIfDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_integers<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  typename thrust::host_vector<int>::iterator h_new_end;
+  typename thrust::device_vector<int>::iterator d_new_end;
+
+  thrust::device_vector<typename thrust::device_vector<int>::iterator> d_new_end_vec(1);
+
+  // test with Predicate that returns a bool
+  {
+    thrust::host_vector<int> h_result(n);
+    thrust::device_vector<int> d_result(n);
+
+    h_new_end = thrust::copy_if(h_data.begin(), h_data.end(), h_result.begin(), is_even<int>());
+
+    copy_if_kernel<<<1, 1>>>(
+      exec, d_data.begin(), d_data.end(), d_result.begin(), is_even<int>(), d_new_end_vec.begin());
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+
+    d_new_end = d_new_end_vec[0];
+
+    h_result.resize(h_new_end - h_result.begin());
+    d_result.resize(d_new_end - d_result.begin());
+
+    ASSERT_EQUAL(h_result, d_result);
+  }
+
+  // test with Predicate that returns a non-bool
+  {
+    thrust::host_vector<int> h_result(n);
+    thrust::device_vector<int> d_result(n);
+
+    h_new_end = thrust::copy_if(h_data.begin(), h_data.end(), h_result.begin(), mod_3<int>());
+
+    copy_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), d_result.begin(), mod_3<int>(), d_new_end_vec.begin());
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+
+    d_new_end = d_new_end_vec[0];
+
+    h_result.resize(h_new_end - h_result.begin());
+    d_result.resize(d_new_end - d_result.begin());
+
+    ASSERT_EQUAL(h_result, d_result);
+  }
+}
+
+void TestCopyIfDeviceSeq()
+{
+  TestCopyIfDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestCopyIfDeviceSeq);
+
+void TestCopyIfDeviceDevice()
+{
+  TestCopyIfDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestCopyIfDeviceDevice);
+
+void TestCopyIfDeviceNoSync()
+{
+  TestCopyIfDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestCopyIfDeviceNoSync);
+#endif
+
+template <typename ExecutionPolicy>
+void TestCopyIfCudaStreams(ExecutionPolicy policy)
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector data{1, 2, 1, 3, 2};
+  Vector result(data.size());
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  Vector::iterator end = thrust::copy_if(policy.on(s), data.begin(), data.end(), result.begin(), is_even<int>());
+
+  ASSERT_EQUAL(end - result.begin(), 2);
+  result.resize(end - result.begin());
+  Vector ref{2, 2};
+  ASSERT_EQUAL(result, ref);
+
+  cudaStreamDestroy(s);
+}
+
+void TestCopyIfCudaStreamsSync()
+{
+  TestCopyIfCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestCopyIfCudaStreamsSync);
+
+void TestCopyIfCudaStreamsNoSync()
+{
+  TestCopyIfCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestCopyIfCudaStreamsNoSync);
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Predicate,
+          typename Iterator4>
+__global__ void copy_if_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first,
+  Iterator1 last,
+  Iterator2 stencil_first,
+  Iterator3 result1,
+  Predicate pred,
+  Iterator4 result2)
+{
+  *result2 = thrust::copy_if(exec, first, last, stencil_first, result1, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestCopyIfStencilDevice(ExecutionPolicy exec)
+{
+  size_t n = 1000;
+  thrust::host_vector<int> h_data(n);
+  thrust::sequence(h_data.begin(), h_data.end());
+  thrust::device_vector<int> d_data(n);
+  thrust::sequence(d_data.begin(), d_data.end());
+
+  thrust::host_vector<int> h_stencil   = unittest::random_integers<int>(n);
+  thrust::device_vector<int> d_stencil = unittest::random_integers<int>(n);
+
+  typename thrust::host_vector<int>::iterator h_new_end;
+  typename thrust::device_vector<int>::iterator d_new_end;
+
+  thrust::device_vector<typename thrust::device_vector<int>::iterator> d_new_end_vec(1);
+
+  // test with Predicate that returns a bool
+  {
+    thrust::host_vector<int> h_result(n);
+    thrust::device_vector<int> d_result(n);
+
+    h_new_end = thrust::copy_if(h_data.begin(), h_data.end(), h_result.begin(), is_even<int>());
+
+    copy_if_kernel<<<1, 1>>>(
+      exec, d_data.begin(), d_data.end(), d_result.begin(), is_even<int>(), d_new_end_vec.begin());
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+
+    d_new_end = d_new_end_vec[0];
+
+    h_result.resize(h_new_end - h_result.begin());
+    d_result.resize(d_new_end - d_result.begin());
+
+    ASSERT_EQUAL(h_result, d_result);
+  }
+
+  // test with Predicate that returns a non-bool
+  {
+    thrust::host_vector<int> h_result(n);
+    thrust::device_vector<int> d_result(n);
+
+    h_new_end = thrust::copy_if(h_data.begin(), h_data.end(), h_result.begin(), mod_3<int>());
+
+    copy_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), d_result.begin(), mod_3<int>(), d_new_end_vec.begin());
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+
+    d_new_end = d_new_end_vec[0];
+
+    h_result.resize(h_new_end - h_result.begin());
+    d_result.resize(d_new_end - d_result.begin());
+
+    ASSERT_EQUAL(h_result, d_result);
+  }
+}
+
+void TestCopyIfStencilDeviceSeq()
+{
+  TestCopyIfStencilDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestCopyIfStencilDeviceSeq);
+
+void TestCopyIfStencilDeviceDevice()
+{
+  TestCopyIfStencilDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestCopyIfStencilDeviceDevice);
+
+void TestCopyIfStencilDeviceNoSync()
+{
+  TestCopyIfStencilDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestCopyIfStencilDeviceNoSync);
+#endif
+
+template <typename ExecutionPolicy>
+void TestCopyIfStencilCudaStreams(ExecutionPolicy policy)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{1, 2, 1, 3, 2};
+
+  Vector result(5);
+
+  Vector stencil{0, 1, 0, 0, 1};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  Vector::iterator end =
+    thrust::copy_if(policy.on(s), data.begin(), data.end(), stencil.begin(), result.begin(), ::cuda::std::identity{});
+
+  ASSERT_EQUAL(end - result.begin(), 2);
+  result.resize(end - result.begin());
+
+  Vector ref{2, 2};
+  ASSERT_EQUAL(result, ref);
+
+  cudaStreamDestroy(s);
+}
+
+void TestCopyIfStencilCudaStreamsSync()
+{
+  TestCopyIfStencilCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestCopyIfStencilCudaStreamsSync);
+
+void TestCopyIfStencilCudaStreamsNoSync()
+{
+  TestCopyIfStencilCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestCopyIfStencilCudaStreamsNoSync);
+
+void TestCopyIfWithMagnitude(int magnitude)
+{
+  using offset_t = std::size_t;
+
+  // Prepare input
+  offset_t num_items = offset_t{1ull} << magnitude;
+  thrust::counting_iterator<offset_t> begin(offset_t{0});
+  auto end = begin + static_cast<std::ptrdiff_t>(num_items);
+  ASSERT_EQUAL(static_cast<offset_t>(::cuda::std::distance(begin, end)), num_items);
+
+  // Run algorithm on large number of items
+  offset_t match_every_nth     = 1000000;
+  offset_t expected_num_copied = (num_items + match_every_nth - 1) / match_every_nth;
+  thrust::device_vector<offset_t> copied_out(expected_num_copied);
+  auto selected_out_end = thrust::copy_if(begin, end, copied_out.begin(), mod_n<offset_t>{match_every_nth});
+
+  // Ensure number of selected items are correct
+  offset_t num_selected_out = static_cast<offset_t>(::cuda::std::distance(copied_out.begin(), selected_out_end));
+  ASSERT_EQUAL(num_selected_out, expected_num_copied);
+  copied_out.resize(expected_num_copied);
+
+  // Ensure selected items are correct
+  auto expected_out_it     = thrust::make_transform_iterator(begin, multiply_n<offset_t>{match_every_nth});
+  bool all_results_correct = thrust::equal(copied_out.begin(), copied_out.end(), expected_out_it);
+  ASSERT_EQUAL(all_results_correct, true);
+}
+
+void TestCopyIfWithLargeNumberOfItems()
+{
+  TestCopyIfWithMagnitude(30);
+  TestCopyIfWithMagnitude(31);
+  TestCopyIfWithMagnitude(32);
+  TestCopyIfWithMagnitude(33);
+}
+DECLARE_UNITTEST(TestCopyIfWithLargeNumberOfItems);
+
+void TestCopyIfStencilWithMagnitude(int magnitude)
+{
+  using offset_t = std::size_t;
+
+  // Prepare input
+  offset_t num_items = offset_t{1ull} << magnitude;
+  thrust::counting_iterator<offset_t> begin(offset_t{0});
+  auto end = begin + static_cast<std::ptrdiff_t>(num_items);
+  thrust::counting_iterator<offset_t> stencil(offset_t{0});
+  ASSERT_EQUAL(static_cast<offset_t>(::cuda::std::distance(begin, end)), num_items);
+
+  // Run algorithm on large number of items
+  offset_t match_every_nth     = 1000000;
+  offset_t expected_num_copied = (num_items + match_every_nth - 1) / match_every_nth;
+  thrust::device_vector<offset_t> copied_out(expected_num_copied);
+  auto selected_out_end = thrust::copy_if(begin, end, stencil, copied_out.begin(), mod_n<offset_t>{match_every_nth});
+
+  // Ensure number of selected items are correct
+  offset_t num_selected_out = static_cast<offset_t>(::cuda::std::distance(copied_out.begin(), selected_out_end));
+  ASSERT_EQUAL(num_selected_out, expected_num_copied);
+  copied_out.resize(expected_num_copied);
+
+  // Ensure selected items are correct
+  auto expected_out_it     = thrust::make_transform_iterator(begin, multiply_n<offset_t>{match_every_nth});
+  bool all_results_correct = thrust::equal(copied_out.begin(), copied_out.end(), expected_out_it);
+  ASSERT_EQUAL(all_results_correct, true);
+}
+
+void TestCopyIfStencilWithLargeNumberOfItems()
+{
+  TestCopyIfStencilWithMagnitude(30);
+  TestCopyIfStencilWithMagnitude(31);
+  TestCopyIfStencilWithMagnitude(32);
+  TestCopyIfStencilWithMagnitude(33);
+}
+DECLARE_UNITTEST(TestCopyIfStencilWithLargeNumberOfItems);

cccl_upstream/thrust/testing/cuda/count.cu

@@ -0,0 +1,103 @@
+#include <thrust/count.h>
+#include <thrust/execution_policy.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename T, typename Iterator2>
+__global__ void count_kernel(ExecutionPolicy exec, Iterator first, Iterator last, T value, Iterator2 result)
+{
+  *result = thrust::count(exec, first, last, value);
+}
+
+template <typename T, typename ExecutionPolicy>
+void TestCountDevice(ExecutionPolicy exec, const size_t n)
+{
+  thrust::host_vector<T> h_data   = unittest::random_samples<T>(n);
+  thrust::device_vector<T> d_data = h_data;
+
+  thrust::device_vector<size_t> d_result(1);
+
+  size_t h_result = thrust::count(h_data.begin(), h_data.end(), T(5));
+
+  count_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), T(5), d_result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(h_result, d_result[0]);
+}
+
+template <typename T>
+void TestCountDeviceSeq(const size_t n)
+{
+  TestCountDevice<T>(thrust::seq, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestCountDeviceSeq);
+
+template <typename T>
+void TestCountDeviceDevice(const size_t n)
+{
+  TestCountDevice<T>(thrust::device, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestCountDeviceDevice);
+
+template <typename ExecutionPolicy, typename Iterator, typename Predicate, typename Iterator2>
+__global__ void count_if_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Predicate pred, Iterator2 result)
+{
+  *result = thrust::count_if(exec, first, last, pred);
+}
+
+template <typename T>
+struct greater_than_five
+{
+  _CCCL_HOST_DEVICE bool operator()(const T& x) const
+  {
+    return x > 5;
+  }
+};
+
+template <typename T, typename ExecutionPolicy>
+void TestCountIfDevice(ExecutionPolicy exec, const size_t n)
+{
+  thrust::host_vector<T> h_data   = unittest::random_samples<T>(n);
+  thrust::device_vector<T> d_data = h_data;
+
+  thrust::device_vector<size_t> d_result(1);
+
+  size_t h_result = thrust::count_if(h_data.begin(), h_data.end(), greater_than_five<T>());
+  count_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), greater_than_five<T>(), d_result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(h_result, d_result[0]);
+}
+
+template <typename T>
+void TestCountIfDeviceSeq(const size_t n)
+{
+  TestCountIfDevice<T>(thrust::seq, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestCountIfDeviceSeq);
+
+template <typename T>
+void TestCountIfDeviceDevice(const size_t n)
+{
+  TestCountIfDevice<T>(thrust::device, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestCountIfDeviceDevice);
+#endif
+
+void TestCountCudaStreams()
+{
+  thrust::device_vector<int> data{1, 1, 0, 0, 1};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  ASSERT_EQUAL(thrust::count(thrust::cuda::par.on(s), data.begin(), data.end(), 0), 2);
+  ASSERT_EQUAL(thrust::count(thrust::cuda::par.on(s), data.begin(), data.end(), 1), 3);
+  ASSERT_EQUAL(thrust::count(thrust::cuda::par.on(s), data.begin(), data.end(), 2), 0);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestCountCudaStreams);

cccl_upstream/thrust/testing/cuda/cudart.cu

@@ -0,0 +1,16 @@
+#include <cuda/std/memory>
+
+#include <cuda_runtime_api.h>
+
+#include <unittest/unittest.h>
+
+template <typename T>
+void TestCudaMallocResultAligned(const std::size_t n)
+{
+  T* ptr = nullptr;
+  cudaMalloc(&ptr, n * sizeof(T));
+  cudaFree(ptr);
+
+  ASSERT_EQUAL(true, ::cuda::std::is_sufficiently_aligned<alignof(T)>(ptr));
+}
+DECLARE_VARIABLE_UNITTEST(TestCudaMallocResultAligned);

cccl_upstream/thrust/testing/cuda/device_buffer_algorithms.cu

@@ -0,0 +1,47 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/execution_policy.h>
+#include <thrust/random.h>
+#include <thrust/sequence.h>
+#include <thrust/shuffle.h>
+#include <thrust/sort.h>
+
+#include <cuda/buffer>
+#include <cuda/cccl_runtime_test_helper.cuh>
+#include <cuda/std/initializer_list>
+#include <cuda/stream>
+
+#include <unittest/unittest.h>
+
+void TestDeviceBufferShuffleCudaStreams()
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto buffer       = cuda::make_device_buffer<int>(stream, device, 5, cuda::no_init);
+  const auto policy = thrust::cuda::par_nosync.on(stream.get());
+
+  thrust::sequence(policy, buffer.begin(), buffer.end());
+
+  thrust::default_random_engine rng{2};
+  thrust::shuffle(policy, buffer.begin(), buffer.end(), rng);
+  thrust::sort(policy, buffer.begin(), buffer.end());
+
+  test_runtime::assert_equal(stream, buffer, {0, 1, 2, 3, 4});
+}
+DECLARE_UNITTEST(TestDeviceBufferShuffleCudaStreams);
+
+void TestDeviceBufferSortCudaStreams()
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto buffer       = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{3, 1, 4, 0, 2});
+  const auto policy = thrust::cuda::par_nosync.on(stream.get());
+
+  thrust::sort(policy, buffer.begin(), buffer.end());
+
+  test_runtime::assert_equal(stream, buffer, {0, 1, 2, 3, 4});
+}
+DECLARE_UNITTEST(TestDeviceBufferSortCudaStreams);

cccl_upstream/thrust/testing/cuda/device_side_universal_vector.cu

@@ -0,0 +1,76 @@
+#include <thrust/universal_vector.h>
+
+#include <unittest/unittest.h>
+
+template <class VecInT, class VecOutT>
+_CCCL_HOST_DEVICE void universal_vector_access(VecInT& in, VecOutT& out)
+{
+  const int expected_front = 4;
+  const int expected_back  = 2;
+
+  out[0] = in.size() == 2 && //
+           in[0] == expected_front && //
+           in.front() == expected_front && //
+         *in.data() == expected_front && //
+           in[1] == expected_back && //
+           in.back() == expected_back;
+}
+
+#if defined(THRUST_TEST_DEVICE_SIDE)
+template <class VecInT, class VecOutT>
+__global__ void universal_vector_device_access_kernel(VecInT& vec, VecOutT& out)
+{
+  universal_vector_access(vec, out);
+}
+
+template <class VecInT, class VecOutT>
+void test_universal_vector_access(VecInT& vec, VecOutT& out)
+{
+  universal_vector_device_access_kernel<<<1, 1>>>(vec, out);
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+  ASSERT_EQUAL(out[0], true);
+}
+#else
+template <class VecInT, class VecOutT>
+void test_universal_vector_access(VecInT& vec, VecOutT& out)
+{
+  universal_vector_access(vec, out);
+  ASSERT_EQUAL(out[0], true);
+}
+#endif
+
+template <typename UniversalIntVector, typename UniversalBoolVector>
+void TestDeviceAccess()
+{
+  using in_vector_t  = UniversalIntVector;
+  using out_vector_t = UniversalBoolVector;
+
+  in_vector_t* in_ptr{};
+  cudaMallocManaged(&in_ptr, sizeof(*in_ptr));
+  new (in_ptr) in_vector_t(1);
+
+  auto& in = *in_ptr;
+  in.resize(2);
+  in = {4, 2};
+
+  out_vector_t* out_ptr{};
+  cudaMallocManaged(&out_ptr, sizeof(*out_ptr));
+  new (out_ptr) out_vector_t(1);
+  auto& out = *out_ptr;
+
+  out.resize(1);
+  out[0] = false;
+
+  test_universal_vector_access(in, out);
+  const auto& const_in = *in_ptr;
+  test_universal_vector_access(const_in, out);
+
+  cudaFree(in_ptr);
+  cudaFree(out_ptr);
+}
+DECLARE_UNITTEST_WITH_NAME((TestDeviceAccess<thrust::universal_vector<int>, thrust::universal_vector<bool>>),
+                           TestUniversalVectorDeviceAccess);
+DECLARE_UNITTEST_WITH_NAME(
+  (TestDeviceAccess<thrust::universal_host_pinned_vector<int>, thrust::universal_host_pinned_vector<bool>>),
+  TestUniversalHPVectorDeviceAccess);

cccl_upstream/thrust/testing/cuda/equal.cu

@@ -0,0 +1,120 @@
+#include <thrust/equal.h>
+#include <thrust/execution_policy.h>
+#include <thrust/functional.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Iterator3>
+__global__ void equal_kernel(ExecutionPolicy exec, Iterator1 first1, Iterator1 last1, Iterator2 first2, Iterator3 result)
+{
+  *result = thrust::equal(exec, first1, last1, first2);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename BinaryPredicate, typename Iterator3>
+__global__ void equal_kernel(
+  ExecutionPolicy exec, Iterator1 first1, Iterator1 last1, Iterator2 first2, BinaryPredicate pred, Iterator3 result)
+{
+  *result = thrust::equal(exec, first1, last1, first2, pred);
+}
+
+template <typename T, typename ExecutionPolicy>
+void TestEqualDevice(ExecutionPolicy exec, const size_t n)
+{
+  thrust::device_vector<T> d_data1 = unittest::random_samples<T>(n);
+  thrust::device_vector<T> d_data2 = unittest::random_samples<T>(n);
+  thrust::device_vector<bool> d_result(1, false);
+
+  // empty ranges
+  equal_kernel<<<1, 1>>>(exec, d_data1.begin(), d_data1.begin(), d_data1.begin(), d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(d_result[0], true);
+
+  // symmetric cases
+  equal_kernel<<<1, 1>>>(exec, d_data1.begin(), d_data1.end(), d_data1.begin(), d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(d_result[0], true);
+
+  if (n > 0)
+  {
+    d_data1[0] = 0;
+    d_data2[0] = 1;
+
+    // different vectors
+    equal_kernel<<<1, 1>>>(exec, d_data1.begin(), d_data1.end(), d_data2.begin(), d_result.begin());
+    {
+      cudaError_t const err = cudaDeviceSynchronize();
+      ASSERT_EQUAL(cudaSuccess, err);
+    }
+
+    ASSERT_EQUAL(d_result[0], false);
+
+    // different predicates
+    equal_kernel<<<1, 1>>>(
+      exec, d_data1.begin(), d_data1.begin() + 1, d_data2.begin(), ::cuda::std::less<T>(), d_result.begin());
+    {
+      cudaError_t const err = cudaDeviceSynchronize();
+      ASSERT_EQUAL(cudaSuccess, err);
+    }
+
+    ASSERT_EQUAL(d_result[0], true);
+
+    equal_kernel<<<1, 1>>>(
+      exec, d_data1.begin(), d_data1.begin() + 1, d_data2.begin(), ::cuda::std::greater<T>(), d_result.begin());
+    {
+      cudaError_t const err = cudaDeviceSynchronize();
+      ASSERT_EQUAL(cudaSuccess, err);
+    }
+
+    ASSERT_EQUAL(d_result[0], false);
+  }
+}
+
+template <typename T>
+void TestEqualDeviceSeq(const size_t n)
+{
+  TestEqualDevice<T>(thrust::seq, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestEqualDeviceSeq);
+
+template <typename T>
+void TestEqualDeviceDevice(const size_t n)
+{
+  TestEqualDevice<T>(thrust::device, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestEqualDeviceDevice);
+#endif
+
+void TestEqualCudaStreams()
+{
+  thrust::device_vector<int> v1 = {5, 2, 0, 0, 0};
+  thrust::device_vector<int> v2 = {5, 2, 0, 6, 1};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  ASSERT_EQUAL(thrust::equal(thrust::cuda::par.on(s), v1.begin(), v1.end(), v1.begin()), true);
+  ASSERT_EQUAL(thrust::equal(thrust::cuda::par.on(s), v1.begin(), v1.end(), v2.begin()), false);
+  ASSERT_EQUAL(thrust::equal(thrust::cuda::par.on(s), v2.begin(), v2.end(), v2.begin()), true);
+
+  ASSERT_EQUAL(thrust::equal(thrust::cuda::par.on(s), v1.begin(), v1.begin() + 0, v1.begin()), true);
+  ASSERT_EQUAL(thrust::equal(thrust::cuda::par.on(s), v1.begin(), v1.begin() + 1, v1.begin()), true);
+  ASSERT_EQUAL(thrust::equal(thrust::cuda::par.on(s), v1.begin(), v1.begin() + 3, v2.begin()), true);
+  ASSERT_EQUAL(thrust::equal(thrust::cuda::par.on(s), v1.begin(), v1.begin() + 4, v2.begin()), false);
+
+  ASSERT_EQUAL(thrust::equal(thrust::cuda::par.on(s), v1.begin(), v1.end(), v2.begin(), ::cuda::std::less_equal<int>()),
+               true);
+  ASSERT_EQUAL(thrust::equal(thrust::cuda::par.on(s), v1.begin(), v1.end(), v2.begin(), ::cuda::std::greater<int>()),
+               false);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestEqualCudaStreams);

cccl_upstream/thrust/testing/cuda/fill.cu

@@ -0,0 +1,207 @@
+#include <thrust/execution_policy.h>
+#include <thrust/fill.h>
+
+#include <algorithm>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename T>
+__global__ void fill_kernel(ExecutionPolicy exec, Iterator first, Iterator last, T value)
+{
+  thrust::fill(exec, first, last, value);
+}
+
+template <typename T, typename ExecutionPolicy>
+void TestFillDevice(ExecutionPolicy exec, size_t n)
+{
+  thrust::host_vector<T> h_data   = unittest::random_integers<T>(n);
+  thrust::device_vector<T> d_data = h_data;
+
+  thrust::fill(h_data.begin() + std::min((size_t) 1, n), h_data.begin() + std::min((size_t) 3, n), (T) 0);
+
+  fill_kernel<<<1, 1>>>(exec, d_data.begin() + std::min((size_t) 1, n), d_data.begin() + std::min((size_t) 3, n), (T) 0);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_data, d_data);
+
+  thrust::fill(h_data.begin() + std::min((size_t) 117, n), h_data.begin() + std::min((size_t) 367, n), (T) 1);
+
+  fill_kernel<<<1, 1>>>(
+    exec, d_data.begin() + std::min((size_t) 117, n), d_data.begin() + std::min((size_t) 367, n), (T) 1);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_data, d_data);
+
+  thrust::fill(h_data.begin() + std::min((size_t) 8, n), h_data.begin() + std::min((size_t) 259, n), (T) 2);
+
+  fill_kernel<<<1, 1>>>(
+    exec, d_data.begin() + std::min((size_t) 8, n), d_data.begin() + std::min((size_t) 259, n), (T) 2);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_data, d_data);
+
+  thrust::fill(h_data.begin() + std::min((size_t) 3, n), h_data.end(), (T) 3);
+
+  fill_kernel<<<1, 1>>>(exec, d_data.begin() + std::min((size_t) 3, n), d_data.end(), (T) 3);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_data, d_data);
+
+  thrust::fill(h_data.begin(), h_data.end(), (T) 4);
+
+  fill_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), (T) 4);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_data, d_data);
+}
+
+template <typename T>
+void TestFillDeviceSeq(size_t n)
+{
+  TestFillDevice<T>(thrust::seq, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestFillDeviceSeq);
+
+template <typename T>
+void TestFillDeviceDevice(size_t n)
+{
+  TestFillDevice<T>(thrust::device, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestFillDeviceDevice);
+
+template <typename ExecutionPolicy, typename Iterator, typename Size, typename T>
+__global__ void fill_n_kernel(ExecutionPolicy exec, Iterator first, Size n, T value)
+{
+  thrust::fill_n(exec, first, n, value);
+}
+
+template <typename T, typename ExecutionPolicy>
+void TestFillNDevice(ExecutionPolicy exec, size_t n)
+{
+  thrust::host_vector<T> h_data   = unittest::random_integers<T>(n);
+  thrust::device_vector<T> d_data = h_data;
+
+  size_t begin_offset = std::min<size_t>(1, n);
+
+  thrust::fill_n(h_data.begin() + begin_offset, std::min((size_t) 3, n) - begin_offset, (T) 0);
+
+  fill_n_kernel<<<1, 1>>>(exec, d_data.begin() + begin_offset, std::min((size_t) 3, n) - begin_offset, (T) 0);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_data, d_data);
+
+  begin_offset = std::min<size_t>(117, n);
+
+  thrust::fill_n(h_data.begin() + begin_offset, std::min((size_t) 367, n) - begin_offset, (T) 1);
+
+  fill_n_kernel<<<1, 1>>>(exec, d_data.begin() + begin_offset, std::min((size_t) 367, n) - begin_offset, (T) 1);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_data, d_data);
+
+  begin_offset = std::min<size_t>(8, n);
+
+  thrust::fill_n(h_data.begin() + begin_offset, std::min((size_t) 259, n) - begin_offset, (T) 2);
+
+  fill_n_kernel<<<1, 1>>>(exec, d_data.begin() + begin_offset, std::min((size_t) 259, n) - begin_offset, (T) 2);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_data, d_data);
+
+  begin_offset = std::min<size_t>(3, n);
+
+  thrust::fill_n(h_data.begin() + begin_offset, h_data.size() - begin_offset, (T) 3);
+
+  fill_n_kernel<<<1, 1>>>(exec, d_data.begin() + begin_offset, d_data.size() - begin_offset, (T) 3);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_data, d_data);
+
+  thrust::fill_n(h_data.begin(), h_data.size(), (T) 4);
+
+  fill_n_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.size(), (T) 4);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_data, d_data);
+}
+
+template <typename T>
+void TestFillNDeviceSeq(size_t n)
+{
+  TestFillNDevice<T>(thrust::seq, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestFillNDeviceSeq);
+
+template <typename T>
+void TestFillNDeviceDevice(size_t n)
+{
+  TestFillNDevice<T>(thrust::device, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestFillNDeviceDevice);
+#endif
+
+void TestFillCudaStreams()
+{
+  thrust::device_vector<int> v{0, 1, 2, 3, 4};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::fill(thrust::cuda::par.on(s), v.begin() + 1, v.begin() + 4, 7);
+  cudaStreamSynchronize(s);
+
+  thrust::device_vector<int> ref{0, 7, 7, 7, 4};
+  ASSERT_EQUAL(v, ref);
+
+  thrust::fill(thrust::cuda::par.on(s), v.begin() + 0, v.begin() + 3, 8);
+  cudaStreamSynchronize(s);
+
+  ref = {8, 8, 8, 7, 4};
+  ASSERT_EQUAL(v, ref);
+
+  thrust::fill(thrust::cuda::par.on(s), v.begin() + 2, v.end(), 9);
+  cudaStreamSynchronize(s);
+
+  ref = {8, 8, 9, 9, 9};
+  ASSERT_EQUAL(v, ref);
+
+  thrust::fill(thrust::cuda::par.on(s), v.begin(), v.end(), 1);
+  cudaStreamSynchronize(s);
+
+  ref = {1, 1, 1, 1, 1};
+  ASSERT_EQUAL(v, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestFillCudaStreams);

cccl_upstream/thrust/testing/cuda/find.cu

@@ -0,0 +1,197 @@
+#include <thrust/execution_policy.h>
+#include <thrust/find.h>
+#include <thrust/functional.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename T, typename Iterator2>
+__global__ void find_kernel(ExecutionPolicy exec, Iterator first, Iterator last, T value, Iterator2 result)
+{
+  *result = thrust::find(exec, first, last, value);
+}
+
+template <typename ExecutionPolicy>
+void TestFindDevice(ExecutionPolicy exec)
+{
+  size_t n = 100;
+
+  thrust::host_vector<int> h_data   = unittest::random_integers<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  typename thrust::host_vector<int>::iterator h_iter;
+
+  using iter_type = typename thrust::device_vector<int>::iterator;
+  thrust::device_vector<iter_type> d_result(1);
+
+  h_iter = thrust::find(h_data.begin(), h_data.end(), int(0));
+
+  find_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), int(0), d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_iter - h_data.begin(), (iter_type) d_result[0] - d_data.begin());
+
+  for (size_t i = 1; i < n; i *= 2)
+  {
+    int sample = h_data[i];
+
+    h_iter = thrust::find(h_data.begin(), h_data.end(), sample);
+
+    find_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), sample, d_result.begin());
+    {
+      cudaError_t const err = cudaDeviceSynchronize();
+      ASSERT_EQUAL(cudaSuccess, err);
+    }
+
+    ASSERT_EQUAL(h_iter - h_data.begin(), (iter_type) d_result[0] - d_data.begin());
+  }
+}
+
+void TestFindDeviceSeq()
+{
+  TestFindDevice(thrust::seq);
+};
+DECLARE_UNITTEST(TestFindDeviceSeq);
+
+void TestFindDeviceDevice()
+{
+  TestFindDevice(thrust::device);
+};
+DECLARE_UNITTEST(TestFindDeviceDevice);
+
+template <typename ExecutionPolicy, typename Iterator, typename Predicate, typename Iterator2>
+__global__ void find_if_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Predicate pred, Iterator2 result)
+{
+  *result = thrust::find_if(exec, first, last, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestFindIfDevice(ExecutionPolicy exec)
+{
+  size_t n = 100;
+
+  thrust::host_vector<int> h_data   = unittest::random_integers<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  typename thrust::host_vector<int>::iterator h_iter;
+
+  using iter_type = typename thrust::device_vector<int>::iterator;
+  thrust::device_vector<iter_type> d_result(1);
+
+  using thrust::placeholders::_1;
+  h_iter = thrust::find_if(h_data.begin(), h_data.end(), _1 == 0);
+
+  find_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), _1 == 0, d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_iter - h_data.begin(), (iter_type) d_result[0] - d_data.begin());
+
+  for (size_t i = 1; i < n; i *= 2)
+  {
+    int sample = h_data[i];
+
+    h_iter = thrust::find_if(h_data.begin(), h_data.end(), _1 == sample);
+
+    find_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), _1 == sample, d_result.begin());
+    {
+      cudaError_t const err = cudaDeviceSynchronize();
+      ASSERT_EQUAL(cudaSuccess, err);
+    }
+
+    ASSERT_EQUAL(h_iter - h_data.begin(), (iter_type) d_result[0] - d_data.begin());
+  }
+}
+
+void TestFindIfDeviceSeq()
+{
+  TestFindIfDevice(thrust::seq);
+};
+DECLARE_UNITTEST(TestFindIfDeviceSeq);
+
+void TestFindIfDeviceDevice()
+{
+  TestFindIfDevice(thrust::device);
+};
+DECLARE_UNITTEST(TestFindIfDeviceDevice);
+
+template <typename ExecutionPolicy, typename Iterator, typename Predicate, typename Iterator2>
+__global__ void find_if_not_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Predicate pred, Iterator2 result)
+{
+  *result = thrust::find_if_not(exec, first, last, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestFindIfNotDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 100;
+  thrust::host_vector<int> h_data   = unittest::random_integers<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  typename thrust::host_vector<int>::iterator h_iter;
+
+  using iter_type = typename thrust::device_vector<int>::iterator;
+  thrust::device_vector<iter_type> d_result(1);
+
+  using thrust::placeholders::_1;
+  h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), _1 != 0);
+
+  find_if_not_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), _1 != 0, d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_iter - h_data.begin(), (iter_type) d_result[0] - d_data.begin());
+
+  for (size_t i = 1; i < n; i *= 2)
+  {
+    int sample = h_data[i];
+
+    h_iter = thrust::find_if_not(h_data.begin(), h_data.end(), _1 != sample);
+
+    find_if_not_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), _1 != sample, d_result.begin());
+    {
+      cudaError_t const err = cudaDeviceSynchronize();
+      ASSERT_EQUAL(cudaSuccess, err);
+    }
+
+    ASSERT_EQUAL(h_iter - h_data.begin(), (iter_type) d_result[0] - d_data.begin());
+  }
+}
+
+void TestFindIfNotDeviceSeq()
+{
+  TestFindIfNotDevice(thrust::seq);
+};
+DECLARE_UNITTEST(TestFindIfNotDeviceSeq);
+
+void TestFindIfNotDeviceDevice()
+{
+  TestFindIfNotDevice(thrust::device);
+};
+DECLARE_UNITTEST(TestFindIfNotDeviceDevice);
+#endif
+
+void TestFindCudaStreams()
+{
+  thrust::device_vector<int> vec{1, 2, 3, 3, 5};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  ASSERT_EQUAL(thrust::find(thrust::cuda::par.on(s), vec.begin(), vec.end(), 0) - vec.begin(), 5);
+  ASSERT_EQUAL(thrust::find(thrust::cuda::par.on(s), vec.begin(), vec.end(), 1) - vec.begin(), 0);
+  ASSERT_EQUAL(thrust::find(thrust::cuda::par.on(s), vec.begin(), vec.end(), 2) - vec.begin(), 1);
+  ASSERT_EQUAL(thrust::find(thrust::cuda::par.on(s), vec.begin(), vec.end(), 3) - vec.begin(), 2);
+  ASSERT_EQUAL(thrust::find(thrust::cuda::par.on(s), vec.begin(), vec.end(), 4) - vec.begin(), 5);
+  ASSERT_EQUAL(thrust::find(thrust::cuda::par.on(s), vec.begin(), vec.end(), 5) - vec.begin(), 4);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestFindCudaStreams);

cccl_upstream/thrust/testing/cuda/for_each.cu

@@ -0,0 +1,238 @@
+#include <thrust/execution_policy.h>
+#include <thrust/for_each.h>
+
+#include <algorithm>
+
+#include "thrust/device_vector.h"
+#include <unittest/unittest.h>
+
+static const size_t NUM_REGISTERS = 64;
+
+template <size_t N>
+_CCCL_HOST_DEVICE void f(int* x)
+{
+  int temp = *x;
+  f<N - 1>(x + 1);
+  *x = temp;
+};
+template <>
+_CCCL_HOST_DEVICE void f<0>(int* /*x*/)
+{}
+template <size_t N>
+struct CopyFunctorWithManyRegisters
+{
+  _CCCL_HOST_DEVICE void operator()(int* ptr)
+  {
+    f<N>(ptr);
+  }
+};
+
+void TestForEachLargeRegisterFootprint()
+{
+  int current_device = -1;
+  cudaGetDevice(&current_device);
+  cudaDeviceProp prop;
+  cudaGetDeviceProperties(&prop, current_device);
+
+  thrust::device_vector<int> data(NUM_REGISTERS, 12345);
+
+  thrust::device_vector<int*> input(1, thrust::raw_pointer_cast(&data[0])); // length is irrelevant
+
+  thrust::for_each(input.begin(), input.end(), CopyFunctorWithManyRegisters<NUM_REGISTERS>());
+}
+DECLARE_UNITTEST(TestForEachLargeRegisterFootprint);
+
+void TestForEachNLargeRegisterFootprint()
+{
+  int current_device = -1;
+  cudaGetDevice(&current_device);
+  cudaDeviceProp prop;
+  cudaGetDeviceProperties(&prop, current_device);
+
+  thrust::device_vector<int> data(NUM_REGISTERS, 12345);
+
+  thrust::device_vector<int*> input(1, thrust::raw_pointer_cast(&data[0])); // length is irrelevant
+
+  thrust::for_each_n(input.begin(), input.size(), CopyFunctorWithManyRegisters<NUM_REGISTERS>());
+}
+DECLARE_UNITTEST(TestForEachNLargeRegisterFootprint);
+
+template <typename T>
+struct mark_present_for_each
+{
+  T* ptr;
+  _CCCL_HOST_DEVICE void operator()(T x)
+  {
+    ptr[(int) x] = 1;
+  }
+};
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename Function>
+__global__ void for_each_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Function f)
+{
+  thrust::for_each(exec, first, last, f);
+}
+
+template <typename T>
+void TestForEachDeviceSeq(const size_t n)
+{
+  const size_t output_size = std::min((size_t) 10, 2 * n);
+
+  thrust::host_vector<T> h_input = unittest::random_integers<T>(n);
+
+  for (size_t i = 0; i < n; i++)
+  {
+    h_input[i] = ((size_t) h_input[i]) % output_size;
+  }
+
+  thrust::device_vector<T> d_input = h_input;
+
+  thrust::host_vector<T> h_output(output_size, (T) 0);
+  thrust::device_vector<T> d_output(output_size, (T) 0);
+
+  mark_present_for_each<T> h_f;
+  mark_present_for_each<T> d_f;
+  h_f.ptr = &h_output[0];
+  d_f.ptr = (&d_output[0]).get();
+
+  thrust::for_each(h_input.begin(), h_input.end(), h_f);
+
+  for_each_kernel<<<1, 1>>>(thrust::seq, d_input.begin(), d_input.end(), d_f);
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(h_output, d_output);
+}
+DECLARE_VARIABLE_UNITTEST(TestForEachDeviceSeq);
+
+template <typename T>
+void TestForEachDeviceDevice(const size_t n)
+{
+  const size_t output_size = std::min((size_t) 10, 2 * n);
+
+  thrust::host_vector<T> h_input = unittest::random_integers<T>(n);
+
+  for (size_t i = 0; i < n; i++)
+  {
+    h_input[i] = ((size_t) h_input[i]) % output_size;
+  }
+
+  thrust::device_vector<T> d_input = h_input;
+
+  thrust::host_vector<T> h_output(output_size, (T) 0);
+  thrust::device_vector<T> d_output(output_size, (T) 0);
+
+  mark_present_for_each<T> h_f;
+  mark_present_for_each<T> d_f;
+  h_f.ptr = &h_output[0];
+  d_f.ptr = (&d_output[0]).get();
+
+  thrust::for_each(h_input.begin(), h_input.end(), h_f);
+
+  for_each_kernel<<<1, 1>>>(thrust::device, d_input.begin(), d_input.end(), d_f);
+  {
+    cudaError_t const err = cudaGetLastError();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_output, d_output);
+}
+DECLARE_VARIABLE_UNITTEST(TestForEachDeviceDevice);
+
+template <typename ExecutionPolicy, typename Iterator, typename Size, typename Function>
+__global__ void for_each_n_kernel(ExecutionPolicy exec, Iterator first, Size n, Function f)
+{
+  thrust::for_each_n(exec, first, n, f);
+}
+
+template <typename T>
+void TestForEachNDeviceSeq(const size_t n)
+{
+  const size_t output_size = std::min((size_t) 10, 2 * n);
+
+  thrust::host_vector<T> h_input = unittest::random_integers<T>(n);
+
+  for (size_t i = 0; i < n; i++)
+  {
+    h_input[i] = static_cast<T>(((size_t) h_input[i]) % output_size);
+  }
+
+  thrust::device_vector<T> d_input = h_input;
+
+  thrust::host_vector<T> h_output(output_size, (T) 0);
+  thrust::device_vector<T> d_output(output_size, (T) 0);
+
+  mark_present_for_each<T> h_f;
+  mark_present_for_each<T> d_f;
+  h_f.ptr = &h_output[0];
+  d_f.ptr = (&d_output[0]).get();
+
+  thrust::for_each_n(h_input.begin(), h_input.size(), h_f);
+
+  for_each_n_kernel<<<1, 1>>>(thrust::seq, d_input.begin(), d_input.size(), d_f);
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(h_output, d_output);
+}
+DECLARE_VARIABLE_UNITTEST(TestForEachNDeviceSeq);
+
+template <typename T>
+void TestForEachNDeviceDevice(const size_t n)
+{
+  const size_t output_size = std::min((size_t) 10, 2 * n);
+
+  thrust::host_vector<T> h_input = unittest::random_integers<T>(n);
+
+  for (size_t i = 0; i < n; i++)
+  {
+    h_input[i] = static_cast<T>(((size_t) h_input[i]) % output_size);
+  }
+
+  thrust::device_vector<T> d_input = h_input;
+
+  thrust::host_vector<T> h_output(output_size, (T) 0);
+  thrust::device_vector<T> d_output(output_size, (T) 0);
+
+  mark_present_for_each<T> h_f;
+  mark_present_for_each<T> d_f;
+  h_f.ptr = &h_output[0];
+  d_f.ptr = (&d_output[0]).get();
+
+  thrust::for_each_n(h_input.begin(), h_input.size(), h_f);
+
+  for_each_n_kernel<<<1, 1>>>(thrust::device, d_input.begin(), d_input.size(), d_f);
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(h_output, d_output);
+}
+DECLARE_VARIABLE_UNITTEST(TestForEachNDeviceDevice);
+#endif
+
+void TestForEachCudaStreams()
+{
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::device_vector<int> input{3, 2, 3, 4, 6};
+  thrust::device_vector<int> output(7, 0);
+
+  mark_present_for_each<int> f;
+  f.ptr = thrust::raw_pointer_cast(output.data());
+
+  thrust::for_each(thrust::cuda::par.on(s), input.begin(), input.end(), f);
+
+  cudaStreamSynchronize(s);
+
+  thrust::device_vector<int> ref{0, 0, 1, 1, 1, 0, 1};
+  ASSERT_EQUAL(output, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestForEachCudaStreams);

cccl_upstream/thrust/testing/cuda/gather.cu

@@ -0,0 +1,202 @@
+#include <thrust/execution_policy.h>
+#include <thrust/gather.h>
+
+#include <algorithm>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Iterator3>
+__global__ void
+gather_kernel(ExecutionPolicy exec, Iterator1 map_first, Iterator1 map_last, Iterator2 elements_first, Iterator3 result)
+{
+  thrust::gather(exec, map_first, map_last, elements_first, result);
+}
+
+template <typename T, typename ExecutionPolicy>
+void TestGatherDevice(ExecutionPolicy exec, const size_t n)
+{
+  const size_t source_size = std::min((size_t) 10, 2 * n);
+
+  // source vectors to gather from
+  thrust::host_vector<T> h_source   = unittest::random_samples<T>(source_size);
+  thrust::device_vector<T> d_source = h_source;
+
+  // gather indices
+  thrust::host_vector<unsigned int> h_map = unittest::random_integers<unsigned int>(n);
+
+  for (size_t i = 0; i < n; i++)
+  {
+    h_map[i] = h_map[i] % source_size;
+  }
+
+  thrust::device_vector<unsigned int> d_map = h_map;
+
+  // gather destination
+  thrust::host_vector<T> h_output(n);
+  thrust::device_vector<T> d_output(n);
+
+  thrust::gather(h_map.begin(), h_map.end(), h_source.begin(), h_output.begin());
+
+  gather_kernel<<<1, 1>>>(exec, d_map.begin(), d_map.end(), d_source.begin(), d_output.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_output, d_output);
+}
+
+template <typename T>
+void TestGatherDeviceSeq(const size_t n)
+{
+  TestGatherDevice<T>(thrust::seq, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestGatherDeviceSeq);
+
+template <typename T>
+void TestGatherDeviceDevice(const size_t n)
+{
+  TestGatherDevice<T>(thrust::device, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestGatherDeviceDevice);
+#endif
+
+void TestGatherCudaStreams()
+{
+  thrust::device_vector<int> map = {6, 2, 1, 7, 2}; // gather indices
+  thrust::device_vector<int> src = {0, 1, 2, 3, 4, 5, 6, 7}; // source vector
+  thrust::device_vector<int> dst = {0, 0, 0, 0, 0}; // destination vector
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::gather(thrust::cuda::par.on(s), map.begin(), map.end(), src.begin(), dst.begin());
+  cudaStreamSynchronize(s);
+
+  thrust::device_vector<int> ref = {6, 2, 1, 7, 2}; // destination vector
+
+  ASSERT_EQUAL(dst, ref);
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestGatherCudaStreams);
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Iterator4,
+          typename Predicate>
+__global__ void gather_if_kernel(
+  ExecutionPolicy exec,
+  Iterator1 map_first,
+  Iterator1 map_last,
+  Iterator2 stencil_first,
+  Iterator3 elements_first,
+  Iterator4 result,
+  Predicate pred)
+{
+  thrust::gather_if(exec, map_first, map_last, stencil_first, elements_first, result, pred);
+}
+
+template <typename T>
+struct is_even_gather_if
+{
+  _CCCL_HOST_DEVICE bool operator()(const T i) const
+  {
+    return (i % 2) == 0;
+  }
+};
+
+template <typename T, typename ExecutionPolicy>
+void TestGatherIfDevice(ExecutionPolicy exec, const size_t n)
+{
+  const size_t source_size = std::min((size_t) 10, 2 * n);
+
+  // source vectors to gather from
+  thrust::host_vector<T> h_source   = unittest::random_samples<T>(source_size);
+  thrust::device_vector<T> d_source = h_source;
+
+  // gather indices
+  thrust::host_vector<unsigned int> h_map = unittest::random_integers<unsigned int>(n);
+
+  for (size_t i = 0; i < n; i++)
+  {
+    h_map[i] = h_map[i] % source_size;
+  }
+
+  thrust::device_vector<unsigned int> d_map = h_map;
+
+  // gather stencil
+  thrust::host_vector<unsigned int> h_stencil = unittest::random_integers<unsigned int>(n);
+
+  for (size_t i = 0; i < n; i++)
+  {
+    h_stencil[i] = h_stencil[i] % 2;
+  }
+
+  thrust::device_vector<unsigned int> d_stencil = h_stencil;
+
+  // gather destination
+  thrust::host_vector<T> h_output(n);
+  thrust::device_vector<T> d_output(n);
+
+  thrust::gather_if(
+    h_map.begin(),
+    h_map.end(),
+    h_stencil.begin(),
+    h_source.begin(),
+    h_output.begin(),
+    is_even_gather_if<unsigned int>());
+
+  gather_if_kernel<<<1, 1>>>(
+    exec,
+    d_map.begin(),
+    d_map.end(),
+    d_stencil.begin(),
+    d_source.begin(),
+    d_output.begin(),
+    is_even_gather_if<unsigned int>());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_output, d_output);
+}
+
+template <typename T>
+void TestGatherIfDeviceSeq(const size_t n)
+{
+  TestGatherIfDevice<T>(thrust::seq, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestGatherIfDeviceSeq);
+
+template <typename T>
+void TestGatherIfDeviceDevice(const size_t n)
+{
+  TestGatherIfDevice<T>(thrust::device, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestGatherIfDeviceDevice);
+#endif
+
+void TestGatherIfCudaStreams()
+{
+  thrust::device_vector<int> flg{0, 1, 0, 1, 0}; // predicate array
+  thrust::device_vector<int> map{6, 2, 1, 7, 2}; // gather indices
+  thrust::device_vector<int> src{0, 1, 2, 3, 4, 5, 6, 7}; // source vector
+  thrust::device_vector<int> dst(5, 0); // destination vector
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::gather_if(thrust::cuda::par.on(s), map.begin(), map.end(), flg.begin(), src.begin(), dst.begin());
+  cudaStreamSynchronize(s);
+
+  thrust::device_vector<int> ref{0, 2, 0, 7, 0}; // destination vector
+
+  ASSERT_EQUAL(dst, ref);
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestGatherIfCudaStreams);

cccl_upstream/thrust/testing/cuda/generate.cu

@@ -0,0 +1,152 @@
+#include <thrust/execution_policy.h>
+#include <thrust/generate.h>
+
+#include <unittest/unittest.h>
+
+template <typename T>
+struct return_value
+{
+  T val;
+
+  return_value() = default;
+  return_value(T v)
+      : val(v)
+  {}
+
+  _CCCL_HOST_DEVICE T operator()()
+  {
+    return val;
+  }
+};
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename Function>
+__global__ void generate_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Function f)
+{
+  thrust::generate(exec, first, last, f);
+}
+
+template <typename T, typename ExecutionPolicy>
+void TestGenerateDevice(ExecutionPolicy exec, const size_t n)
+{
+  thrust::host_vector<T> h_result(n);
+  thrust::device_vector<T> d_result(n);
+
+  T value = 13;
+  return_value<T> f(value);
+
+  thrust::generate(h_result.begin(), h_result.end(), f);
+
+  generate_kernel<<<1, 1>>>(exec, d_result.begin(), d_result.end(), f);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_result, d_result);
+}
+
+template <typename T>
+void TestGenerateDeviceSeq(const size_t n)
+{
+  TestGenerateDevice<T>(thrust::seq, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestGenerateDeviceSeq);
+
+template <typename T>
+void TestGenerateDeviceDevice(const size_t n)
+{
+  TestGenerateDevice<T>(thrust::device, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestGenerateDeviceDevice);
+#endif
+
+void TestGenerateCudaStreams()
+{
+  thrust::device_vector<int> result(5);
+
+  int value = 13;
+
+  return_value<int> f(value);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::generate(thrust::cuda::par.on(s), result.begin(), result.end(), f);
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(result[0], value);
+  ASSERT_EQUAL(result[1], value);
+  ASSERT_EQUAL(result[2], value);
+  ASSERT_EQUAL(result[3], value);
+  ASSERT_EQUAL(result[4], value);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestGenerateCudaStreams);
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename Size, typename Function>
+__global__ void generate_n_kernel(ExecutionPolicy exec, Iterator first, Size n, Function f)
+{
+  thrust::generate_n(exec, first, n, f);
+}
+
+template <typename T, typename ExecutionPolicy>
+void TestGenerateNDevice(ExecutionPolicy exec, const size_t n)
+{
+  thrust::host_vector<T> h_result(n);
+  thrust::device_vector<T> d_result(n);
+
+  T value = 13;
+  return_value<T> f(value);
+
+  thrust::generate_n(h_result.begin(), h_result.size(), f);
+
+  generate_n_kernel<<<1, 1>>>(exec, d_result.begin(), d_result.size(), f);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_result, d_result);
+}
+
+template <typename T>
+void TestGenerateNDeviceSeq(const size_t n)
+{
+  TestGenerateNDevice<T>(thrust::seq, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestGenerateNDeviceSeq);
+
+template <typename T>
+void TestGenerateNDeviceDevice(const size_t n)
+{
+  TestGenerateNDevice<T>(thrust::device, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestGenerateNDeviceDevice);
+#endif
+
+void TestGenerateNCudaStreams()
+{
+  thrust::device_vector<int> result(5);
+
+  int value = 13;
+
+  return_value<int> f(value);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::generate_n(thrust::cuda::par.on(s), result.begin(), result.size(), f);
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(result[0], value);
+  ASSERT_EQUAL(result[1], value);
+  ASSERT_EQUAL(result[2], value);
+  ASSERT_EQUAL(result[3], value);
+  ASSERT_EQUAL(result[4], value);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestGenerateNCudaStreams);

cccl_upstream/thrust/testing/cuda/inner_product.cu

@@ -0,0 +1,67 @@
+#include <thrust/execution_policy.h>
+#include <thrust/inner_product.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename T, typename Iterator3>
+__global__ void inner_product_kernel(
+  ExecutionPolicy exec, Iterator1 first1, Iterator1 last1, Iterator2 first2, T init, Iterator3 result)
+{
+  *result = thrust::inner_product(exec, first1, last1, first2, init);
+}
+
+template <typename ExecutionPolicy>
+void TestInnerProductDevice(ExecutionPolicy exec)
+{
+  size_t n = 1000;
+
+  thrust::host_vector<int> h_v1 = unittest::random_integers<int>(n);
+  thrust::host_vector<int> h_v2 = unittest::random_integers<int>(n);
+
+  thrust::device_vector<int> d_v1 = h_v1;
+  thrust::device_vector<int> d_v2 = h_v2;
+
+  thrust::device_vector<int> result(1);
+
+  int init = 13;
+
+  int expected = thrust::inner_product(h_v1.begin(), h_v1.end(), h_v2.begin(), init);
+
+  inner_product_kernel<<<1, 1>>>(exec, d_v1.begin(), d_v1.end(), d_v2.begin(), init, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(expected, result[0]);
+}
+
+void TestInnerProductDeviceSeq()
+{
+  TestInnerProductDevice(thrust::seq);
+};
+DECLARE_UNITTEST(TestInnerProductDeviceSeq);
+
+void TestInnerProductDeviceDevice()
+{
+  TestInnerProductDevice(thrust::device);
+};
+DECLARE_UNITTEST(TestInnerProductDeviceDevice);
+#endif
+
+void TestInnerProductCudaStreams()
+{
+  thrust::device_vector<int> v1 = {1, -2, 3};
+  thrust::device_vector<int> v2 = {-4, 5, 6};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  int init   = 3;
+  int result = thrust::inner_product(thrust::cuda::par.on(s), v1.begin(), v1.end(), v2.begin(), init);
+  ASSERT_EQUAL(result, 7);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestInnerProductCudaStreams);

cccl_upstream/thrust/testing/cuda/is_partitioned.cu

@@ -0,0 +1,135 @@
+#include <thrust/execution_policy.h>
+#include <thrust/functional.h>
+#include <thrust/partition.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename Predicate, typename Iterator2>
+__global__ void
+is_partitioned_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Predicate pred, Iterator2 result)
+{
+  *result = thrust::is_partitioned(exec, first, last, pred);
+}
+
+template <typename T>
+struct is_even
+{
+  _CCCL_HOST_DEVICE bool operator()(T x) const
+  {
+    return ((int) x % 2) == 0;
+  }
+};
+
+template <typename ExecutionPolicy>
+void TestIsPartitionedDevice(ExecutionPolicy exec)
+{
+  size_t n = 1000;
+
+  n = ::cuda::std::max<size_t>(n, 2);
+
+  thrust::device_vector<int> v = unittest::random_integers<int>(n);
+
+  thrust::device_vector<bool> result(1);
+
+  v[0] = 1;
+  v[1] = 0;
+
+  is_partitioned_kernel<<<1, 1>>>(exec, v.begin(), v.end(), is_even<int>(), result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(false, result[0]);
+
+  thrust::partition(v.begin(), v.end(), is_even<int>());
+
+  is_partitioned_kernel<<<1, 1>>>(exec, v.begin(), v.end(), is_even<int>(), result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(true, result[0]);
+}
+
+void TestIsPartitionedDeviceSeq()
+{
+  TestIsPartitionedDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestIsPartitionedDeviceSeq);
+
+void TestIsPartitionedDeviceDevice()
+{
+  TestIsPartitionedDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestIsPartitionedDeviceDevice);
+#endif
+
+void TestIsPartitionedCudaStreams()
+{
+  thrust::device_vector<int> v(4);
+  v[0] = 1;
+  v[1] = 1;
+  v[2] = 1;
+  v[3] = 0;
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  // empty partition
+  ASSERT_EQUAL_QUIET(true,
+                     thrust::is_partitioned(thrust::cuda::par.on(s), v.begin(), v.begin(), ::cuda::std::identity{}));
+
+  // one element true partition
+  ASSERT_EQUAL_QUIET(
+    true, thrust::is_partitioned(thrust::cuda::par.on(s), v.begin(), v.begin() + 1, ::cuda::std::identity{}));
+
+  // just true partition
+  ASSERT_EQUAL_QUIET(
+    true, thrust::is_partitioned(thrust::cuda::par.on(s), v.begin(), v.begin() + 2, ::cuda::std::identity{}));
+
+  // both true & false partitions
+  ASSERT_EQUAL_QUIET(true,
+                     thrust::is_partitioned(thrust::cuda::par.on(s), v.begin(), v.end(), ::cuda::std::identity{}));
+
+  // one element false partition
+  ASSERT_EQUAL_QUIET(true,
+                     thrust::is_partitioned(thrust::cuda::par.on(s), v.begin() + 3, v.end(), ::cuda::std::identity{}));
+
+  v[0] = 1;
+  v[1] = 0;
+  v[2] = 1;
+  v[3] = 1;
+
+  // not partitioned
+  ASSERT_EQUAL_QUIET(false,
+                     thrust::is_partitioned(thrust::cuda::par.on(s), v.begin(), v.end(), ::cuda::std::identity{}));
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestIsPartitionedCudaStreams);
+
+template <typename T>
+struct is_even_non_const
+{
+  _CCCL_HOST_DEVICE bool operator()(T x) // no const
+  {
+    return ((int) x % 2) == 0;
+  }
+};
+
+void TestIsPartitionedWithNonConstPredicate()
+{
+  thrust::device_vector<int> partitioned   = {0, 2, 4, 1, 3, 5};
+  thrust::device_vector<int> unpartitioned = {0, 1, 2, 3};
+
+  ASSERT_EQUAL_QUIET(
+    true, thrust::is_partitioned(thrust::cuda::par, partitioned.begin(), partitioned.end(), is_even_non_const<int>{}));
+
+  ASSERT_EQUAL_QUIET(
+    false,
+    thrust::is_partitioned(thrust::cuda::par, unpartitioned.begin(), unpartitioned.end(), is_even_non_const<int>{}));
+}
+DECLARE_UNITTEST(TestIsPartitionedWithNonConstPredicate);

cccl_upstream/thrust/testing/cuda/is_sorted.cu

@@ -0,0 +1,93 @@
+#include <thrust/execution_policy.h>
+#include <thrust/sort.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename Iterator2>
+__global__ void is_sorted_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Iterator2 result)
+{
+  *result = thrust::is_sorted(exec, first, last);
+}
+
+template <typename ExecutionPolicy>
+void TestIsSortedDevice(ExecutionPolicy exec)
+{
+  size_t n = 1000;
+
+  thrust::device_vector<int> v = unittest::random_integers<int>(n);
+
+  thrust::device_vector<bool> result(1);
+
+  v[0] = 1;
+  v[1] = 0;
+
+  is_sorted_kernel<<<1, 1>>>(exec, v.begin(), v.end(), result.begin());
+
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(false, result[0]);
+
+  thrust::sort(v.begin(), v.end());
+
+  is_sorted_kernel<<<1, 1>>>(exec, v.begin(), v.end(), result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(true, result[0]);
+}
+
+void TestIsSortedDeviceSeq()
+{
+  TestIsSortedDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestIsSortedDeviceSeq);
+
+void TestIsSortedDeviceDevice()
+{
+  TestIsSortedDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestIsSortedDeviceDevice);
+#endif
+
+void TestIsSortedCudaStreams()
+{
+  thrust::device_vector<int> v(4);
+  v[0] = 0;
+  v[1] = 5;
+  v[2] = 8;
+  v[3] = 0;
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  ASSERT_EQUAL(thrust::is_sorted(thrust::cuda::par.on(s), v.begin(), v.begin() + 0), true);
+  ASSERT_EQUAL(thrust::is_sorted(thrust::cuda::par.on(s), v.begin(), v.begin() + 1), true);
+
+  // the following line crashes gcc 4.3
+#if (__GNUC__ == 4) && (__GNUC_MINOR__ == 3)
+  // do nothing
+#else
+  // compile this line on other compilers
+  ASSERT_EQUAL(thrust::is_sorted(thrust::cuda::par.on(s), v.begin(), v.begin() + 2), true);
+#endif // GCC
+
+  ASSERT_EQUAL(thrust::is_sorted(thrust::cuda::par.on(s), v.begin(), v.begin() + 3), true);
+  ASSERT_EQUAL(thrust::is_sorted(thrust::cuda::par.on(s), v.begin(), v.begin() + 4), false);
+
+  ASSERT_EQUAL(thrust::is_sorted(thrust::cuda::par.on(s), v.begin(), v.begin() + 3, ::cuda::std::less<int>()), true);
+
+  ASSERT_EQUAL(thrust::is_sorted(thrust::cuda::par.on(s), v.begin(), v.begin() + 1, ::cuda::std::greater<int>()), true);
+  ASSERT_EQUAL(thrust::is_sorted(thrust::cuda::par.on(s), v.begin(), v.begin() + 4, ::cuda::std::greater<int>()),
+               false);
+
+  ASSERT_EQUAL(thrust::is_sorted(thrust::cuda::par.on(s), v.begin(), v.end()), false);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestIsSortedCudaStreams);

cccl_upstream/thrust/testing/cuda/is_sorted_until.cu

@@ -0,0 +1,120 @@
+#include <thrust/execution_policy.h>
+#include <thrust/sort.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
+__global__ void is_sorted_until_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result)
+{
+  *result = thrust::is_sorted_until(exec, first, last);
+}
+
+template <typename ExecutionPolicy>
+void TestIsSortedUntilDevice(ExecutionPolicy exec)
+{
+  size_t n = 1000;
+
+  thrust::device_vector<int> v = unittest::random_integers<int>(n);
+
+  using iter_type = typename thrust::device_vector<int>::iterator;
+
+  thrust::device_vector<iter_type> result(1);
+
+  v[0] = 1;
+  v[1] = 0;
+
+  is_sorted_until_kernel<<<1, 1>>>(exec, v.begin(), v.end(), result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL_QUIET(v.begin() + 1, (iter_type) result[0]);
+
+  thrust::sort(v.begin(), v.end());
+
+  is_sorted_until_kernel<<<1, 1>>>(exec, v.begin(), v.end(), result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL_QUIET(v.end(), (iter_type) result[0]);
+}
+
+void TestIsSortedUntilDeviceSeq()
+{
+  TestIsSortedUntilDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestIsSortedUntilDeviceSeq);
+
+void TestIsSortedUntilDeviceDevice()
+{
+  TestIsSortedUntilDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestIsSortedUntilDeviceDevice);
+#endif
+
+void TestIsSortedUntilCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+
+  using T        = Vector::value_type;
+  using Iterator = Vector::iterator;
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  Vector v(4);
+  v[0] = 0;
+  v[1] = 5;
+  v[2] = 8;
+  v[3] = 0;
+
+  Iterator first = v.begin();
+
+  Iterator last = v.begin() + 0;
+  Iterator ref  = last;
+  ASSERT_EQUAL_QUIET(ref, thrust::is_sorted_until(thrust::cuda::par.on(s), first, last));
+
+  last = v.begin() + 1;
+  ref  = last;
+  ASSERT_EQUAL_QUIET(ref, thrust::is_sorted_until(thrust::cuda::par.on(s), first, last));
+
+  last = v.begin() + 2;
+  ref  = last;
+  ASSERT_EQUAL_QUIET(ref, thrust::is_sorted_until(thrust::cuda::par.on(s), first, last));
+
+  last = v.begin() + 3;
+  ref  = v.begin() + 3;
+  ASSERT_EQUAL_QUIET(ref, thrust::is_sorted_until(thrust::cuda::par.on(s), first, last));
+
+  last = v.begin() + 4;
+  ref  = v.begin() + 3;
+  ASSERT_EQUAL_QUIET(ref, thrust::is_sorted_until(thrust::cuda::par.on(s), first, last));
+
+  last = v.begin() + 3;
+  ref  = v.begin() + 3;
+  ASSERT_EQUAL_QUIET(ref, thrust::is_sorted_until(thrust::cuda::par.on(s), first, last, ::cuda::std::less<T>()));
+
+  last = v.begin() + 4;
+  ref  = v.begin() + 3;
+  ASSERT_EQUAL_QUIET(ref, thrust::is_sorted_until(thrust::cuda::par.on(s), first, last, ::cuda::std::less<T>()));
+
+  last = v.begin() + 1;
+  ref  = v.begin() + 1;
+  ASSERT_EQUAL_QUIET(ref, thrust::is_sorted_until(thrust::cuda::par.on(s), first, last, ::cuda::std::greater<T>()));
+
+  last = v.begin() + 4;
+  ref  = v.begin() + 1;
+  ASSERT_EQUAL_QUIET(ref, thrust::is_sorted_until(thrust::cuda::par.on(s), first, last, ::cuda::std::greater<T>()));
+
+  first = v.begin() + 2;
+  last  = v.begin() + 4;
+  ref   = v.begin() + 4;
+  ASSERT_EQUAL_QUIET(ref, thrust::is_sorted_until(thrust::cuda::par.on(s), first, last, ::cuda::std::greater<T>()));
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestIsSortedUntilCudaStreams);

cccl_upstream/thrust/testing/cuda/logical.cu

@@ -0,0 +1,316 @@
+#include <thrust/execution_policy.h>
+#include <thrust/functional.h>
+#include <thrust/logical.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename Function, typename Iterator2>
+__global__ void all_of_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Function f, Iterator2 result)
+{
+  *result = thrust::all_of(exec, first, last, f);
+}
+
+template <typename ExecutionPolicy>
+void TestAllOfDevice(ExecutionPolicy exec)
+{
+  using T = int;
+  thrust::device_vector<T> v(3, 1);
+  thrust::device_vector<bool> result(1);
+
+  all_of_kernel<<<1, 1>>>(exec, v.begin(), v.end(), ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(true, result[0]);
+
+  v[1] = 0;
+
+  all_of_kernel<<<1, 1>>>(exec, v.begin(), v.end(), ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(false, result[0]);
+
+  all_of_kernel<<<1, 1>>>(exec, v.begin() + 0, v.begin() + 0, ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(true, result[0]);
+
+  all_of_kernel<<<1, 1>>>(exec, v.begin() + 0, v.begin() + 1, ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(true, result[0]);
+
+  all_of_kernel<<<1, 1>>>(exec, v.begin() + 0, v.begin() + 2, ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(false, result[0]);
+
+  all_of_kernel<<<1, 1>>>(exec, v.begin() + 1, v.begin() + 2, ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(false, result[0]);
+}
+
+void TestAllOfDeviceSeq()
+{
+  TestAllOfDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestAllOfDeviceSeq);
+
+void TestAllOfDeviceDevice()
+{
+  TestAllOfDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestAllOfDeviceDevice);
+#endif
+
+void TestAllOfCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector v(3, T{1});
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  ASSERT_EQUAL(thrust::all_of(thrust::cuda::par.on(s), v.begin(), v.end(), ::cuda::std::identity{}), true);
+
+  v[1] = 0;
+
+  ASSERT_EQUAL(thrust::all_of(thrust::cuda::par.on(s), v.begin(), v.end(), ::cuda::std::identity{}), false);
+
+  ASSERT_EQUAL(thrust::all_of(thrust::cuda::par.on(s), v.begin() + 0, v.begin() + 0, ::cuda::std::identity{}), true);
+  ASSERT_EQUAL(thrust::all_of(thrust::cuda::par.on(s), v.begin() + 0, v.begin() + 1, ::cuda::std::identity{}), true);
+  ASSERT_EQUAL(thrust::all_of(thrust::cuda::par.on(s), v.begin() + 0, v.begin() + 2, ::cuda::std::identity{}), false);
+  ASSERT_EQUAL(thrust::all_of(thrust::cuda::par.on(s), v.begin() + 1, v.begin() + 2, ::cuda::std::identity{}), false);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestAllOfCudaStreams);
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename Function, typename Iterator2>
+__global__ void any_of_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Function f, Iterator2 result)
+{
+  *result = thrust::any_of(exec, first, last, f);
+}
+
+template <typename ExecutionPolicy>
+void TestAnyOfDevice(ExecutionPolicy exec)
+{
+  using T = int;
+
+  thrust::device_vector<T> v(3, 1);
+  thrust::device_vector<bool> result(1);
+
+  any_of_kernel<<<1, 1>>>(exec, v.begin(), v.end(), ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(true, result[0]);
+
+  v[1] = 0;
+
+  any_of_kernel<<<1, 1>>>(exec, v.begin(), v.end(), ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(true, result[0]);
+
+  any_of_kernel<<<1, 1>>>(exec, v.begin() + 0, v.begin() + 0, ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(false, result[0]);
+
+  any_of_kernel<<<1, 1>>>(exec, v.begin() + 0, v.begin() + 1, ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(true, result[0]);
+
+  any_of_kernel<<<1, 1>>>(exec, v.begin() + 0, v.begin() + 2, ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(true, result[0]);
+
+  any_of_kernel<<<1, 1>>>(exec, v.begin() + 1, v.begin() + 2, ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(false, result[0]);
+}
+
+void TestAnyOfDeviceSeq()
+{
+  TestAnyOfDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestAnyOfDeviceSeq);
+
+void TestAnyOfDeviceDevice()
+{
+  TestAnyOfDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestAnyOfDeviceDevice);
+#endif
+
+void TestAnyOfCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector v(3, T{1});
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  ASSERT_EQUAL(thrust::any_of(thrust::cuda::par.on(s), v.begin(), v.end(), ::cuda::std::identity{}), true);
+
+  v[1] = 0;
+
+  ASSERT_EQUAL(thrust::any_of(thrust::cuda::par.on(s), v.begin(), v.end(), ::cuda::std::identity{}), true);
+
+  ASSERT_EQUAL(thrust::any_of(thrust::cuda::par.on(s), v.begin() + 0, v.begin() + 0, ::cuda::std::identity{}), false);
+  ASSERT_EQUAL(thrust::any_of(thrust::cuda::par.on(s), v.begin() + 0, v.begin() + 1, ::cuda::std::identity{}), true);
+  ASSERT_EQUAL(thrust::any_of(thrust::cuda::par.on(s), v.begin() + 0, v.begin() + 2, ::cuda::std::identity{}), true);
+  ASSERT_EQUAL(thrust::any_of(thrust::cuda::par.on(s), v.begin() + 1, v.begin() + 2, ::cuda::std::identity{}), false);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestAnyOfCudaStreams);
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename Function, typename Iterator2>
+__global__ void none_of_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Function f, Iterator2 result)
+{
+  *result = thrust::none_of(exec, first, last, f);
+}
+
+template <typename ExecutionPolicy>
+void TestNoneOfDevice(ExecutionPolicy exec)
+{
+  using T = int;
+
+  thrust::device_vector<T> v(3, 1);
+  thrust::device_vector<bool> result(1);
+
+  none_of_kernel<<<1, 1>>>(exec, v.begin(), v.end(), ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(false, result[0]);
+
+  v[1] = 0;
+
+  none_of_kernel<<<1, 1>>>(exec, v.begin(), v.end(), ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(false, result[0]);
+
+  none_of_kernel<<<1, 1>>>(exec, v.begin() + 0, v.begin() + 0, ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(true, result[0]);
+
+  none_of_kernel<<<1, 1>>>(exec, v.begin() + 0, v.begin() + 1, ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(false, result[0]);
+
+  none_of_kernel<<<1, 1>>>(exec, v.begin() + 0, v.begin() + 2, ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(false, result[0]);
+
+  none_of_kernel<<<1, 1>>>(exec, v.begin() + 1, v.begin() + 2, ::cuda::std::identity{}, result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(true, result[0]);
+}
+
+void TestNoneOfDeviceSeq()
+{
+  TestNoneOfDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestNoneOfDeviceSeq);
+
+void TestNoneOfDeviceDevice()
+{
+  TestNoneOfDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestNoneOfDeviceDevice);
+#endif
+
+void TestNoneOfCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector v(3, T{1});
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  ASSERT_EQUAL(thrust::none_of(thrust::cuda::par.on(s), v.begin(), v.end(), ::cuda::std::identity{}), false);
+
+  v[1] = 0;
+
+  ASSERT_EQUAL(thrust::none_of(thrust::cuda::par.on(s), v.begin(), v.end(), ::cuda::std::identity{}), false);
+
+  ASSERT_EQUAL(thrust::none_of(thrust::cuda::par.on(s), v.begin() + 0, v.begin() + 0, ::cuda::std::identity{}), true);
+  ASSERT_EQUAL(thrust::none_of(thrust::cuda::par.on(s), v.begin() + 0, v.begin() + 1, ::cuda::std::identity{}), false);
+  ASSERT_EQUAL(thrust::none_of(thrust::cuda::par.on(s), v.begin() + 0, v.begin() + 2, ::cuda::std::identity{}), false);
+  ASSERT_EQUAL(thrust::none_of(thrust::cuda::par.on(s), v.begin() + 1, v.begin() + 2, ::cuda::std::identity{}), true);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestNoneOfCudaStreams);

cccl_upstream/thrust/testing/cuda/max_element.cu

@@ -0,0 +1,131 @@
+#include <thrust/execution_policy.h>
+#include <thrust/extrema.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename Iterator2>
+__global__ void max_element_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Iterator2 result)
+{
+  *result = thrust::max_element(exec, first, last);
+}
+
+template <typename ExecutionPolicy, typename Iterator, typename BinaryPredicate, typename Iterator2>
+__global__ void
+max_element_kernel(ExecutionPolicy exec, Iterator first, Iterator last, BinaryPredicate pred, Iterator2 result)
+{
+  *result = thrust::max_element(exec, first, last, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestMaxElementDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  using iter_type = typename thrust::device_vector<int>::iterator;
+
+  thrust::device_vector<iter_type> d_result(1);
+
+  typename thrust::host_vector<int>::iterator h_max = thrust::max_element(h_data.begin(), h_data.end());
+
+  max_element_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_max - h_data.begin(), (iter_type) d_result[0] - d_data.begin());
+
+  typename thrust::host_vector<int>::iterator h_min =
+    thrust::max_element(h_data.begin(), h_data.end(), ::cuda::std::greater<int>());
+
+  max_element_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), ::cuda::std::greater<int>(), d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_min - h_data.begin(), (iter_type) d_result[0] - d_data.begin());
+}
+
+void TestMaxElementDeviceSeq()
+{
+  TestMaxElementDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestMaxElementDeviceSeq);
+
+void TestMaxElementDeviceDevice()
+{
+  TestMaxElementDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestMaxElementDeviceDevice);
+
+void TestMaxElementDeviceNoSync()
+{
+  TestMaxElementDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestMaxElementDeviceNoSync);
+#endif
+
+template <typename ExecutionPolicy>
+void TestMaxElementCudaStreams(ExecutionPolicy policy)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data(6);
+  data[0] = 3;
+  data[1] = 5;
+  data[2] = 1;
+  data[3] = 2;
+  data[4] = 5;
+  data[5] = 1;
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  auto streampolicy = policy.on(s);
+
+  ASSERT_EQUAL(*thrust::max_element(streampolicy, data.begin(), data.end()), 5);
+  ASSERT_EQUAL(thrust::max_element(streampolicy, data.begin(), data.end()) - data.begin(), 1);
+
+  ASSERT_EQUAL(*thrust::max_element(streampolicy, data.begin(), data.end(), ::cuda::std::greater<T>()), 1);
+  ASSERT_EQUAL(thrust::max_element(streampolicy, data.begin(), data.end(), ::cuda::std::greater<T>()) - data.begin(),
+               2);
+
+  cudaStreamDestroy(s);
+}
+
+void TestMaxElementCudaStreamsSync()
+{
+  TestMaxElementCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestMaxElementCudaStreamsSync);
+
+void TestMaxElementCudaStreamsNoSync()
+{
+  TestMaxElementCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestMaxElementCudaStreamsNoSync);
+
+void TestMaxElementDevicePointer()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data(6);
+  data[0] = 3;
+  data[1] = 5;
+  data[2] = 1;
+  data[3] = 2;
+  data[4] = 5;
+  data[5] = 1;
+
+  T* raw_ptr = thrust::raw_pointer_cast(data.data());
+  size_t n   = data.size();
+  ASSERT_EQUAL(thrust::max_element(thrust::device, raw_ptr, raw_ptr + n) - raw_ptr, 1);
+  ASSERT_EQUAL(thrust::max_element(thrust::device, raw_ptr, raw_ptr + n, ::cuda::std::greater<T>()) - raw_ptr, 2);
+}
+DECLARE_UNITTEST(TestMaxElementDevicePointer);

cccl_upstream/thrust/testing/cuda/memory.cu

@@ -0,0 +1,130 @@
+#include <thrust/execution_policy.h>
+#include <thrust/logical.h>
+#include <thrust/memory.h>
+#include <thrust/system/cpp/memory.h>
+#include <thrust/system/cuda/memory.h>
+
+#include <unittest/unittest.h>
+
+template <typename T1, typename T2>
+bool are_same_type(const T1&, const T2&)
+{
+  return false;
+}
+
+template <typename T>
+bool are_same_type(const T&, const T&)
+{
+  return true;
+}
+
+void TestSelectSystemCudaToCpp()
+{
+  using thrust::system::detail::generic::select_system;
+
+  thrust::cuda::tag cuda_tag;
+  thrust::cpp::tag cpp_tag;
+  thrust::cuda_cub::cross_system<thrust::cuda::tag, thrust::cpp::tag> cuda_to_cpp(cuda_tag, cpp_tag);
+
+  // select_system(cuda::tag, thrust::host_system_tag) should return cuda_to_cpp
+  bool is_cuda_to_cpp = are_same_type(cuda_to_cpp, select_system(cuda_tag, cpp_tag));
+  ASSERT_EQUAL(true, is_cuda_to_cpp);
+}
+DECLARE_UNITTEST(TestSelectSystemCudaToCpp);
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename Iterator>
+__global__ void get_temporary_buffer_kernel(size_t n, Iterator result)
+{
+  *result = thrust::get_temporary_buffer<int>(thrust::seq, n);
+}
+
+template <typename Pointer>
+__global__ void return_temporary_buffer_kernel(Pointer ptr, std::ptrdiff_t n)
+{
+  thrust::return_temporary_buffer(thrust::seq, ptr, n);
+}
+
+void TestGetTemporaryBufferDeviceSeq()
+{
+  const std::ptrdiff_t n = 9001;
+
+  using pointer         = thrust::pointer<int, thrust::detail::seq_t>;
+  using ptr_and_sz_type = cuda::std::pair<pointer, std::ptrdiff_t>;
+  thrust::device_vector<ptr_and_sz_type> d_result(1);
+
+  get_temporary_buffer_kernel<<<1, 1>>>(n, d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ptr_and_sz_type ptr_and_sz = d_result[0];
+
+  if (ptr_and_sz.second > 0)
+  {
+    ASSERT_EQUAL(ptr_and_sz.second, n);
+
+    const int ref_val = 13;
+    thrust::device_vector<int> ref(n, ref_val);
+
+    thrust::fill_n(thrust::device, ptr_and_sz.first, n, ref_val);
+
+    ASSERT_EQUAL(
+      true,
+      thrust::all_of(thrust::device, ptr_and_sz.first, ptr_and_sz.first + n, thrust::placeholders::_1 == ref_val));
+
+    return_temporary_buffer_kernel<<<1, 1>>>(ptr_and_sz.first, ptr_and_sz.second);
+    {
+      cudaError_t const err = cudaDeviceSynchronize();
+      ASSERT_EQUAL(cudaSuccess, err);
+    }
+  }
+}
+DECLARE_UNITTEST(TestGetTemporaryBufferDeviceSeq);
+
+template <typename Iterator>
+__global__ void malloc_kernel(size_t n, Iterator result)
+{
+  *result = static_cast<int*>(thrust::malloc(thrust::seq, sizeof(int) * n).get());
+}
+
+template <typename Pointer>
+__global__ void free_kernel(Pointer ptr)
+{
+  thrust::free(thrust::seq, ptr);
+}
+
+void TestMallocDeviceSeq()
+{
+  const std::ptrdiff_t n = 9001;
+
+  using pointer = thrust::pointer<int, thrust::detail::seq_t>;
+  thrust::device_vector<pointer> d_result(1);
+
+  malloc_kernel<<<1, 1>>>(n, d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  pointer ptr = d_result[0];
+
+  if (ptr.get() != 0)
+  {
+    const int ref_val = 13;
+    thrust::device_vector<int> ref(n, ref_val);
+
+    thrust::fill_n(thrust::device, ptr, n, ref_val);
+
+    ASSERT_EQUAL(true, thrust::all_of(thrust::device, ptr, ptr + n, thrust::placeholders::_1 == ref_val));
+
+    free_kernel<<<1, 1>>>(ptr);
+    {
+      cudaError_t const err = cudaDeviceSynchronize();
+      ASSERT_EQUAL(cudaSuccess, err);
+    }
+  }
+}
+DECLARE_UNITTEST(TestMallocDeviceSeq);
+#endif

cccl_upstream/thrust/testing/cuda/merge.cu

@@ -0,0 +1,92 @@
+#include <thrust/execution_policy.h>
+#include <thrust/extrema.h>
+#include <thrust/merge.h>
+#include <thrust/sort.h>
+
+#include <cuda/buffer>
+#include <cuda/cccl_runtime_test_helper.cuh>
+#include <cuda/launch>
+#include <cuda/stream>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+struct merge_kernel
+{
+  template <typename ExecutionPolicy, typename Input1, typename Input2, typename Size, typename Output>
+  __device__ void operator()(ExecutionPolicy exec, Input1 a, Input2 b, Size b_size, Output result) const
+  {
+    const auto end = thrust::merge(exec, a.begin(), a.end(), b.begin(), b.begin() + b_size, result.begin());
+    TEST_ASSERT_DEVICE(end == result.end());
+  }
+};
+
+template <typename ExecutionPolicy>
+void TestMergeDevice(ExecutionPolicy exec)
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  const size_t n         = 10000;
+  const size_t sizes[]   = {0, 1, n / 2, n, n + 1, 2 * n};
+  const size_t num_sizes = sizeof(sizes) / sizeof(size_t);
+
+  const auto max_size = static_cast<size_t>(*thrust::max_element(sizes, sizes + num_sizes));
+  auto h_a            = test_runtime::random_integers_buffer<int, unittest::int8_t>(stream, n);
+  auto h_b            = test_runtime::random_integers_buffer<int, unittest::int8_t>(stream, max_size, n);
+
+  thrust::stable_sort(h_a.begin(), h_a.end());
+  thrust::stable_sort(h_b.begin(), h_b.end());
+
+  auto d_a = cuda::make_device_buffer<int>(stream, device, h_a);
+  auto d_b = cuda::make_device_buffer<int>(stream, device, h_b);
+
+  for (size_t i = 0; i < num_sizes; i++)
+  {
+    const size_t size = sizes[i];
+
+    auto h_result = test_runtime::make_host_buffer<int>(stream, n + size);
+    stream.sync();
+
+    const auto h_end = thrust::merge(h_a.begin(), h_a.end(), h_b.begin(), h_b.begin() + size, h_result.begin());
+    ASSERT_EQUAL_QUIET(h_result.end(), h_end);
+
+    auto result = cuda::make_device_buffer<int>(stream, device, h_result.size(), cuda::no_init);
+
+    cuda::launch(stream, test_runtime::single_thread_config(), merge_kernel{}, exec, d_a, d_b, size, result);
+    stream.sync();
+
+    test_runtime::assert_equal(stream, result, h_result);
+  }
+}
+
+void TestMergeDeviceSeq()
+{
+  TestMergeDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestMergeDeviceSeq);
+
+void TestMergeDeviceDevice()
+{
+  TestMergeDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestMergeDeviceDevice);
+#endif
+
+void TestMergeCudaStreams()
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a      = cuda::make_device_buffer<int>(stream, device, {0, 2, 4});
+  auto b      = cuda::make_device_buffer<int>(stream, device, {0, 3, 3, 4});
+  auto result = cuda::make_device_buffer<int>(stream, device, 7, cuda::no_init);
+
+  const auto end =
+    thrust::merge(thrust::cuda::par.on(stream.get()), a.begin(), a.end(), b.begin(), b.end(), result.begin());
+  stream.sync();
+
+  ASSERT_EQUAL_QUIET(result.end(), end);
+  test_runtime::assert_equal(stream, result, {0, 0, 2, 3, 3, 4, 4});
+}
+DECLARE_UNITTEST(TestMergeCudaStreams);

cccl_upstream/thrust/testing/cuda/merge_by_key.cu

@@ -0,0 +1,119 @@
+#include <thrust/execution_policy.h>
+#include <thrust/merge.h>
+
+#include <cuda/buffer>
+#include <cuda/cccl_runtime_test_helper.cuh>
+#include <cuda/launch>
+#include <cuda/stream>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+struct merge_by_key_kernel
+{
+  template <typename ExecutionPolicy,
+            typename Keys1,
+            typename Keys2,
+            typename Values1,
+            typename Values2,
+            typename KeysOutput,
+            typename ValuesOutput>
+  __device__ void operator()(
+    ExecutionPolicy exec,
+    Keys1 keys1,
+    Keys2 keys2,
+    Values1 values1,
+    Values2 values2,
+    KeysOutput keys_result,
+    ValuesOutput values_result) const
+  {
+    auto end = thrust::merge_by_key(
+      exec,
+      keys1.begin(),
+      keys1.end(),
+      keys2.begin(),
+      keys2.end(),
+      values1.begin(),
+      values2.begin(),
+      keys_result.begin(),
+      values_result.begin());
+    TEST_ASSERT_DEVICE(end.first == keys_result.end());
+    TEST_ASSERT_DEVICE(end.second == values_result.end());
+  }
+};
+
+template <typename ExecutionPolicy>
+void TestMergeByKeyDevice(ExecutionPolicy exec)
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a_key = cuda::make_device_buffer<int>(stream, device, {0, 2, 4});
+  auto a_val = cuda::make_device_buffer<int>(stream, device, {13, 7, 42});
+  auto b_key = cuda::make_device_buffer<int>(stream, device, {0, 3, 3, 4});
+  auto b_val = cuda::make_device_buffer<int>(stream, device, {42, 42, 7, 13});
+
+  auto result_key = cuda::make_device_buffer<int>(stream, device, 7, cuda::no_init);
+  auto result_val = cuda::make_device_buffer<int>(stream, device, 7, cuda::no_init);
+
+  cuda::launch(
+    stream,
+    test_runtime::single_thread_config(),
+    merge_by_key_kernel{},
+    exec,
+    a_key,
+    b_key,
+    a_val,
+    b_val,
+    result_key,
+    result_val);
+  stream.sync();
+
+  test_runtime::assert_equal(stream, result_key, {0, 0, 2, 3, 3, 4, 4});
+  test_runtime::assert_equal(stream, result_val, {13, 42, 7, 42, 7, 42, 13});
+}
+
+void TestMergeByKeyDeviceSeq()
+{
+  TestMergeByKeyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestMergeByKeyDeviceSeq);
+
+void TestMergeByKeyDeviceDevice()
+{
+  TestMergeByKeyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestMergeByKeyDeviceDevice);
+#endif
+
+void TestMergeByKeyCudaStreams()
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a_key = cuda::make_device_buffer<int>(stream, device, {0, 2, 4});
+  auto a_val = cuda::make_device_buffer<int>(stream, device, {13, 7, 42});
+  auto b_key = cuda::make_device_buffer<int>(stream, device, {0, 3, 3, 4});
+  auto b_val = cuda::make_device_buffer<int>(stream, device, {42, 42, 7, 13});
+
+  auto result_key = cuda::make_device_buffer<int>(stream, device, 7, cuda::no_init);
+  auto result_val = cuda::make_device_buffer<int>(stream, device, 7, cuda::no_init);
+
+  const auto end = thrust::merge_by_key(
+    thrust::cuda::par.on(stream.get()),
+    a_key.begin(),
+    a_key.end(),
+    b_key.begin(),
+    b_key.end(),
+    a_val.begin(),
+    b_val.begin(),
+    result_key.begin(),
+    result_val.begin());
+  stream.sync();
+
+  ASSERT_EQUAL_QUIET(result_key.end(), end.first);
+  ASSERT_EQUAL_QUIET(result_val.end(), end.second);
+  test_runtime::assert_equal(stream, result_key, {0, 0, 2, 3, 3, 4, 4});
+  test_runtime::assert_equal(stream, result_val, {13, 42, 7, 42, 7, 42, 13});
+}
+DECLARE_UNITTEST(TestMergeByKeyCudaStreams);

cccl_upstream/thrust/testing/cuda/merge_sort.cu

@@ -0,0 +1,249 @@
+#include <thrust/execution_policy.h>
+#include <thrust/functional.h>
+#include <thrust/sort.h>
+
+#include <unittest/unittest.h>
+
+template <typename T>
+struct less_div_10
+{
+  _CCCL_HOST_DEVICE bool operator()(const T& lhs, const T& rhs) const
+  {
+    return ((int) lhs) / 10 < ((int) rhs) / 10;
+  }
+};
+
+template <class Vector>
+void InitializeSimpleKeySortTest(Vector& unsorted_keys, Vector& sorted_keys)
+{
+  unsorted_keys.resize(7);
+  unsorted_keys[0] = 1;
+  unsorted_keys[1] = 3;
+  unsorted_keys[2] = 6;
+  unsorted_keys[3] = 5;
+  unsorted_keys[4] = 2;
+  unsorted_keys[5] = 0;
+  unsorted_keys[6] = 4;
+
+  sorted_keys.resize(7);
+  sorted_keys[0] = 0;
+  sorted_keys[1] = 1;
+  sorted_keys[2] = 2;
+  sorted_keys[3] = 3;
+  sorted_keys[4] = 4;
+  sorted_keys[5] = 5;
+  sorted_keys[6] = 6;
+}
+
+template <class Vector>
+void InitializeSimpleKeyValueSortTest(
+  Vector& unsorted_keys, Vector& unsorted_values, Vector& sorted_keys, Vector& sorted_values)
+{
+  unsorted_keys.resize(7);
+  unsorted_values.resize(7);
+
+  // clang-format off
+  unsorted_keys[0] = 1;  unsorted_values[0] = 0;
+  unsorted_keys[1] = 3;  unsorted_values[1] = 1;
+  unsorted_keys[2] = 6;  unsorted_values[2] = 2;
+  unsorted_keys[3] = 5;  unsorted_values[3] = 3;
+  unsorted_keys[4] = 2;  unsorted_values[4] = 4;
+  unsorted_keys[5] = 0;  unsorted_values[5] = 5;
+  unsorted_keys[6] = 4;  unsorted_values[6] = 6;
+
+  sorted_keys.resize(7);
+  sorted_values.resize(7);
+  sorted_keys[0] = 0;  sorted_values[1] = 0;
+  sorted_keys[1] = 1;  sorted_values[3] = 1;
+  sorted_keys[2] = 2;  sorted_values[6] = 2;
+  sorted_keys[3] = 3;  sorted_values[5] = 3;
+  sorted_keys[4] = 4;  sorted_values[2] = 4;
+  sorted_keys[5] = 5;  sorted_values[0] = 5;
+  sorted_keys[6] = 6;  sorted_values[4] = 6;
+  // clang-format on
+}
+
+template <class Vector>
+void InitializeSimpleStableKeySortTest(Vector& unsorted_keys, Vector& sorted_keys)
+{
+  unsorted_keys.resize(9);
+  unsorted_keys[0] = 25;
+  unsorted_keys[1] = 14;
+  unsorted_keys[2] = 35;
+  unsorted_keys[3] = 16;
+  unsorted_keys[4] = 26;
+  unsorted_keys[5] = 34;
+  unsorted_keys[6] = 36;
+  unsorted_keys[7] = 24;
+  unsorted_keys[8] = 15;
+
+  sorted_keys.resize(9);
+  sorted_keys[0] = 14;
+  sorted_keys[1] = 16;
+  sorted_keys[2] = 15;
+  sorted_keys[3] = 25;
+  sorted_keys[4] = 26;
+  sorted_keys[5] = 24;
+  sorted_keys[6] = 35;
+  sorted_keys[7] = 34;
+  sorted_keys[8] = 36;
+}
+
+void TestMergeSortKeySimple()
+{
+#if 0
+    using Vector = thrust::device_vector<int>;
+    using T = Vector::value_type;
+
+    Vector unsorted_keys;
+    Vector   sorted_keys;
+
+    InitializeSimpleKeySortTest(unsorted_keys, sorted_keys);
+
+    thrust::cuda_bulk::tag cuda_tag;
+    thrust::system::cuda_bulk::detail::detail::stable_merge_sort(cuda_tag, unsorted_keys.begin(), unsorted_keys.end(), ::cuda::std::less<T>());
+
+    ASSERT_EQUAL(unsorted_keys, sorted_keys);
+#else
+  KNOWN_FAILURE;
+#endif
+}
+DECLARE_UNITTEST(TestMergeSortKeySimple);
+
+void TestMergeSortKeyValueSimple()
+{
+#if 0
+    using Vector = thrust::device_vector<int>;
+    using T = Vector::value_type;
+
+    Vector unsorted_keys, unsorted_values;
+    Vector   sorted_keys,   sorted_values;
+
+    InitializeSimpleKeyValueSortTest(unsorted_keys, unsorted_values, sorted_keys, sorted_values);
+
+    thrust::cuda_bulk::tag cuda_tag;
+    thrust::system::cuda_bulk::detail::detail::stable_merge_sort_by_key(cuda_tag, unsorted_keys.begin(), unsorted_keys.end(), unsorted_values.begin(), ::cuda::std::less<T>());
+
+    ASSERT_EQUAL(unsorted_keys,   sorted_keys);
+    ASSERT_EQUAL(unsorted_values, sorted_values);
+#else
+  KNOWN_FAILURE;
+#endif
+}
+DECLARE_UNITTEST(TestMergeSortKeyValueSimple);
+
+void TestMergeSortStableKeySimple()
+{
+#if 0
+    using Vector = thrust::device_vector<int>;
+    using T = Vector::value_type;
+
+    Vector unsorted_keys;
+    Vector   sorted_keys;
+
+    InitializeSimpleStableKeySortTest(unsorted_keys, sorted_keys);
+
+    thrust::cuda_bulk::tag cuda_tag;
+    thrust::system::cuda_bulk::detail::detail::stable_merge_sort(cuda_tag, unsorted_keys.begin(), unsorted_keys.end(), less_div_10<T>());
+
+    ASSERT_EQUAL(unsorted_keys,   sorted_keys);
+#else
+  KNOWN_FAILURE;
+#endif
+}
+DECLARE_UNITTEST(TestMergeSortStableKeySimple);
+
+void TestMergeSortDescendingKey()
+{
+#if 0
+    const size_t n = 10027;
+
+    thrust::host_vector<int>   h_data = unittest::random_integers<int>(n);
+    thrust::device_vector<int> d_data = h_data;
+
+    thrust::sort(h_data.begin(), h_data.end(), ::cuda::std::greater<int>());
+
+    thrust::cuda_bulk::tag cuda_tag;
+    thrust::system::cuda_bulk::detail::detail::stable_merge_sort(cuda_tag, d_data.begin(), d_data.end(), ::cuda::std::greater<int>());
+
+    ASSERT_EQUAL(h_data, d_data);
+#else
+  KNOWN_FAILURE;
+#endif
+}
+DECLARE_UNITTEST(TestMergeSortDescendingKey);
+
+template <typename T>
+void TestMergeSortAscendingKeyValue([[maybe_unused]] const size_t n)
+{
+#if 0
+    thrust::host_vector<T>   h_keys = unittest::random_integers<T>(n);
+    thrust::device_vector<T> d_keys = h_keys;
+
+    thrust::host_vector<T>   h_values = unittest::random_integers<T>(n);
+    thrust::device_vector<T> d_values = h_values;
+
+    thrust::sort_by_key(h_keys.begin(), h_keys.end(), h_values.begin(), ::cuda::std::less<T>());
+
+    thrust::cuda_bulk::tag cuda_tag;
+    thrust::system::cuda_bulk::detail::detail::stable_merge_sort_by_key(cuda_tag, d_keys.begin(), d_keys.end(), d_values.begin(), ::cuda::std::less<T>());
+
+    ASSERT_EQUAL(h_keys,   d_keys);
+    ASSERT_EQUAL(h_values, d_values);
+#else
+  KNOWN_FAILURE;
+#endif
+}
+DECLARE_VARIABLE_UNITTEST(TestMergeSortAscendingKeyValue);
+
+void TestMergeSortDescendingKeyValue()
+{
+#if 0
+    const size_t n = 10027;
+
+    thrust::host_vector<int>   h_keys = unittest::random_integers<int>(n);
+    thrust::device_vector<int> d_keys = h_keys;
+
+    thrust::host_vector<int>   h_values = unittest::random_integers<int>(n);
+    thrust::device_vector<int> d_values = h_values;
+
+    thrust::sort_by_key(h_keys.begin(), h_keys.end(), h_values.begin(), ::cuda::std::greater<int>());
+
+    thrust::cuda_bulk::tag cuda_tag;
+    thrust::system::cuda_bulk::detail::detail::stable_merge_sort_by_key(cuda_tag, d_keys.begin(), d_keys.end(), d_values.begin(), ::cuda::std::greater<int>());
+
+    ASSERT_EQUAL(h_keys,   d_keys);
+    ASSERT_EQUAL(h_values, d_values);
+#else
+  KNOWN_FAILURE;
+#endif
+}
+DECLARE_UNITTEST(TestMergeSortDescendingKeyValue);
+
+template <typename U>
+void TestMergeSortKeyValue([[maybe_unused]] size_t n)
+{
+#if 0
+  using T = key_value<U,U>;
+
+  thrust::host_vector<U> h_keys   = unittest::random_integers<U>(n);
+  thrust::host_vector<U> h_values = unittest::random_integers<U>(n);
+
+  thrust::host_vector<T> h_data(n);
+  for(size_t i = 0; i < n; ++i)
+  {
+    h_data[i] = T(h_keys[i], h_values[i]);
+  }
+
+  thrust::device_vector<T> d_data = h_data;
+
+  thrust::stable_sort(h_data.begin(), h_data.end());
+  thrust::cuda_bulk::tag cuda_tag;
+  thrust::system::cuda_bulk::detail::detail::stable_merge_sort(cuda_tag, d_data.begin(), d_data.end(), ::cuda::std::less<T>());
+
+  ASSERT_EQUAL_QUIET(h_data, d_data);
+#else
+  KNOWN_FAILURE;
+#endif
+}
+DECLARE_VARIABLE_UNITTEST(TestMergeSortKeyValue);

cccl_upstream/thrust/testing/cuda/min_element.cu

@@ -0,0 +1,112 @@
+#include <thrust/execution_policy.h>
+#include <thrust/extrema.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename Iterator2>
+__global__ void min_element_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Iterator2 result)
+{
+  *result = thrust::min_element(exec, first, last);
+}
+
+template <typename ExecutionPolicy, typename Iterator, typename BinaryPredicate, typename Iterator2>
+__global__ void
+min_element_kernel(ExecutionPolicy exec, Iterator first, Iterator last, BinaryPredicate pred, Iterator2 result)
+{
+  *result = thrust::min_element(exec, first, last, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestMinElementDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  using iter_type = typename thrust::device_vector<int>::iterator;
+
+  thrust::device_vector<iter_type> d_result(1);
+
+  typename thrust::host_vector<int>::iterator h_min = thrust::min_element(h_data.begin(), h_data.end());
+
+  min_element_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_min - h_data.begin(), (iter_type) d_result[0] - d_data.begin());
+
+  typename thrust::host_vector<int>::iterator h_max =
+    thrust::min_element(h_data.begin(), h_data.end(), ::cuda::std::greater<int>());
+
+  min_element_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), ::cuda::std::greater<int>(), d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(h_max - h_data.begin(), (iter_type) d_result[0] - d_data.begin());
+}
+
+void TestMinElementDeviceSeq()
+{
+  TestMinElementDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestMinElementDeviceSeq);
+
+void TestMinElementDeviceDevice()
+{
+  TestMinElementDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestMinElementDeviceDevice);
+#endif
+
+void TestMinElementCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data(6);
+  data[0] = 3;
+  data[1] = 5;
+  data[2] = 1;
+  data[3] = 2;
+  data[4] = 5;
+  data[5] = 1;
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  ASSERT_EQUAL(*thrust::min_element(thrust::cuda::par.on(s), data.begin(), data.end()), 1);
+  ASSERT_EQUAL(thrust::min_element(thrust::cuda::par.on(s), data.begin(), data.end()) - data.begin(), 2);
+
+  ASSERT_EQUAL(*thrust::min_element(thrust::cuda::par.on(s), data.begin(), data.end(), ::cuda::std::greater<T>()), 5);
+  ASSERT_EQUAL(
+    thrust::min_element(thrust::cuda::par.on(s), data.begin(), data.end(), ::cuda::std::greater<T>()) - data.begin(),
+    1);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestMinElementCudaStreams);
+
+void TestMinElementDevicePointer()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data(6);
+  data[0] = 3;
+  data[1] = 5;
+  data[2] = 1;
+  data[3] = 2;
+  data[4] = 5;
+  data[5] = 1;
+
+  T* raw_ptr = thrust::raw_pointer_cast(data.data());
+  size_t n   = data.size();
+  ASSERT_EQUAL(thrust::min_element(thrust::device, raw_ptr, raw_ptr + n) - raw_ptr, 2);
+  ASSERT_EQUAL(thrust::min_element(thrust::device, raw_ptr, raw_ptr + n, ::cuda::std::greater<T>()) - raw_ptr, 1);
+}
+DECLARE_UNITTEST(TestMinElementDevicePointer);

cccl_upstream/thrust/testing/cuda/minmax_element.cu

@@ -0,0 +1,126 @@
+#include <thrust/extrema.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
+__global__ void minmax_element_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result)
+{
+  *result = thrust::minmax_element(exec, first, last);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename BinaryPredicate>
+__global__ void
+minmax_element_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, BinaryPredicate pred, Iterator2 result)
+{
+  *result = thrust::minmax_element(exec, first, last, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestMinMaxElementDevice(ExecutionPolicy exec)
+{
+  size_t n = 1000;
+
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  typename thrust::host_vector<int>::iterator h_min;
+  typename thrust::host_vector<int>::iterator h_max;
+  typename thrust::device_vector<int>::iterator d_min;
+  typename thrust::device_vector<int>::iterator d_max;
+
+  using pair_type =
+    cuda::std::pair<typename thrust::device_vector<int>::iterator, typename thrust::device_vector<int>::iterator>;
+
+  thrust::device_vector<pair_type> d_result(1);
+
+  h_min = thrust::minmax_element(h_data.begin(), h_data.end()).first;
+  h_max = thrust::minmax_element(h_data.begin(), h_data.end()).second;
+
+  d_min = thrust::minmax_element(d_data.begin(), d_data.end()).first;
+  d_max = thrust::minmax_element(d_data.begin(), d_data.end()).second;
+
+  minmax_element_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  d_min = ((pair_type) d_result[0]).first;
+  d_max = ((pair_type) d_result[0]).second;
+
+  ASSERT_EQUAL(h_min - h_data.begin(), d_min - d_data.begin());
+  ASSERT_EQUAL(h_max - h_data.begin(), d_max - d_data.begin());
+
+  h_max = thrust::minmax_element(h_data.begin(), h_data.end(), ::cuda::std::greater<int>()).first;
+  h_min = thrust::minmax_element(h_data.begin(), h_data.end(), ::cuda::std::greater<int>()).second;
+
+  minmax_element_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), ::cuda::std::greater<int>(), d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  d_max = ((pair_type) d_result[0]).first;
+  d_min = ((pair_type) d_result[0]).second;
+
+  ASSERT_EQUAL(h_min - h_data.begin(), d_min - d_data.begin());
+  ASSERT_EQUAL(h_max - h_data.begin(), d_max - d_data.begin());
+}
+
+void TestMinMaxElementDeviceSeq()
+{
+  TestMinMaxElementDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestMinMaxElementDeviceSeq);
+
+void TestMinMaxElementDeviceDevice()
+{
+  TestMinMaxElementDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestMinMaxElementDeviceDevice);
+#endif
+
+void TestMinMaxElementCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector data(6);
+  data[0] = 3;
+  data[1] = 5;
+  data[2] = 1;
+  data[3] = 2;
+  data[4] = 5;
+  data[5] = 1;
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  ASSERT_EQUAL(*thrust::minmax_element(thrust::cuda::par.on(s), data.begin(), data.end()).first, 1);
+  ASSERT_EQUAL(*thrust::minmax_element(thrust::cuda::par.on(s), data.begin(), data.end()).second, 5);
+  ASSERT_EQUAL(thrust::minmax_element(thrust::cuda::par.on(s), data.begin(), data.end()).first - data.begin(), 2);
+  ASSERT_EQUAL(thrust::minmax_element(thrust::cuda::par.on(s), data.begin(), data.end()).second - data.begin(), 1);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestMinMaxElementCudaStreams);
+
+void TestMinMaxElementDevicePointer()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data(6);
+  data[0] = 3;
+  data[1] = 5;
+  data[2] = 1;
+  data[3] = 2;
+  data[4] = 5;
+  data[5] = 1;
+
+  T* raw_ptr = thrust::raw_pointer_cast(data.data());
+  size_t n   = data.size();
+  ASSERT_EQUAL(thrust::minmax_element(thrust::device, raw_ptr, raw_ptr + n).first - raw_ptr, 2);
+  ASSERT_EQUAL(thrust::minmax_element(thrust::device, raw_ptr, raw_ptr + n).second - raw_ptr, 1);
+}
+DECLARE_UNITTEST(TestMinMaxElementDevicePointer);

cccl_upstream/thrust/testing/cuda/mismatch.cu

@@ -0,0 +1,130 @@
+#include <thrust/execution_policy.h>
+#include <thrust/mismatch.h>
+
+#include <cuda/std/atomic>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Iterator3>
+__global__ void
+mismatch_kernel(ExecutionPolicy exec, Iterator1 first1, Iterator1 last1, Iterator2 first2, Iterator3 result)
+{
+  *result = thrust::mismatch(exec, first1, last1, first2);
+}
+
+template <typename ExecutionPolicy>
+void TestMismatchDevice(ExecutionPolicy exec)
+{
+  thrust::device_vector<int> a = {1, 2, 3, 4};
+  thrust::device_vector<int> b = {1, 2, 4, 3};
+
+  using pair_type =
+    cuda::std::pair<typename thrust::device_vector<int>::iterator, typename thrust::device_vector<int>::iterator>;
+
+  thrust::device_vector<pair_type> d_result(1);
+
+  mismatch_kernel<<<1, 1>>>(exec, a.begin(), a.end(), b.begin(), d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(2, ((pair_type) d_result[0]).first - a.begin());
+  ASSERT_EQUAL(2, ((pair_type) d_result[0]).second - b.begin());
+
+  b[2] = 3;
+
+  mismatch_kernel<<<1, 1>>>(exec, a.begin(), a.end(), b.begin(), d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(3, ((pair_type) d_result[0]).first - a.begin());
+  ASSERT_EQUAL(3, ((pair_type) d_result[0]).second - b.begin());
+
+  b[3] = 4;
+
+  mismatch_kernel<<<1, 1>>>(exec, a.begin(), a.end(), b.begin(), d_result.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(4, ((pair_type) d_result[0]).first - a.begin());
+  ASSERT_EQUAL(4, ((pair_type) d_result[0]).second - b.begin());
+}
+
+void TestMismatchDeviceSeq()
+{
+  TestMismatchDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestMismatchDeviceSeq);
+
+void TestMismatchDeviceDevice()
+{
+  TestMismatchDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestMismatchDeviceDevice);
+#endif
+
+void TestMismatchCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector a = {1, 2, 3, 4};
+  Vector b = {1, 2, 4, 3};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  ASSERT_EQUAL(thrust::mismatch(thrust::cuda::par.on(s), a.begin(), a.end(), b.begin()).first - a.begin(), 2);
+  ASSERT_EQUAL(thrust::mismatch(thrust::cuda::par.on(s), a.begin(), a.end(), b.begin()).second - b.begin(), 2);
+
+  b[2] = 3;
+
+  ASSERT_EQUAL(thrust::mismatch(thrust::cuda::par.on(s), a.begin(), a.end(), b.begin()).first - a.begin(), 3);
+  ASSERT_EQUAL(thrust::mismatch(thrust::cuda::par.on(s), a.begin(), a.end(), b.begin()).second - b.begin(), 3);
+
+  b[3] = 4;
+
+  ASSERT_EQUAL(thrust::mismatch(thrust::cuda::par.on(s), a.begin(), a.end(), b.begin()).first - a.begin(), 4);
+  ASSERT_EQUAL(thrust::mismatch(thrust::cuda::par.on(s), a.begin(), a.end(), b.begin()).second - b.begin(), 4);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestMismatchCudaStreams);
+
+// see https://github.com/NVIDIA/cccl/issues/3591
+template <typename T>
+class Wrapper
+{
+public:
+  Wrapper()
+  {
+    ++my_count;
+  }
+
+  _CCCL_HOST_DEVICE bool operator==(const Wrapper&) const
+  {
+    return true;
+  }
+
+  ~Wrapper()
+  {
+    --my_count;
+  }
+
+private:
+  static cuda::std::atomic<size_t> my_count;
+  T dummy;
+};
+
+void TestMismatchBug3591()
+{
+  using T = Wrapper<int32_t>;
+  T* p    = nullptr;
+  thrust::mismatch(thrust::device, p, p, p, cuda::std::equal_to<T>());
+}
+DECLARE_UNITTEST(TestMismatchBug3591);

cccl_upstream/thrust/testing/cuda/offset_iterator.cu

@@ -0,0 +1,125 @@
+#include <thrust/distance.h>
+#include <thrust/functional.h>
+#include <thrust/iterator/offset_iterator.h>
+#include <thrust/iterator/transform_iterator.h>
+
+#include <cuda/std/iterator>
+
+#include <unittest/unittest.h>
+
+struct device_only_iterator
+{
+  using iterator_category = cuda::std::random_access_iterator_tag;
+  using difference_type   = cuda::std::ptrdiff_t;
+  using value_type        = int;
+  using pointer           = int*;
+  using reference         = int&;
+
+  _CCCL_HOST_DEVICE device_only_iterator(pointer ptr)
+      : m_ptr(ptr)
+  {}
+
+  _CCCL_DEVICE reference operator*() const
+  {
+    return *m_ptr;
+  }
+
+  _CCCL_DEVICE device_only_iterator& operator++()
+  {
+    ++m_ptr;
+    return *this;
+  }
+
+  _CCCL_DEVICE device_only_iterator operator++(int)
+  {
+    device_only_iterator tmp = *this;
+    ++*this;
+    return tmp;
+  }
+
+  _CCCL_DEVICE device_only_iterator& operator--()
+  {
+    --m_ptr;
+    return *this;
+  }
+
+  _CCCL_DEVICE device_only_iterator operator--(int)
+  {
+    device_only_iterator tmp = *this;
+    --*this;
+    return tmp;
+  }
+
+  _CCCL_DEVICE device_only_iterator& operator+=(difference_type n)
+  {
+    m_ptr += n;
+    return *this;
+  }
+
+  _CCCL_DEVICE friend device_only_iterator operator+(device_only_iterator it, difference_type d)
+  {
+    return it += d;
+  }
+
+  _CCCL_DEVICE friend difference_type operator-(const device_only_iterator& a, const device_only_iterator& b)
+  {
+    return a.m_ptr - b.m_ptr;
+  }
+
+  _CCCL_DEVICE friend bool operator==(const device_only_iterator& a, const device_only_iterator& b)
+  {
+    return a.m_ptr == b.m_ptr;
+  }
+
+  _CCCL_DEVICE friend bool operator!=(const device_only_iterator& a, const device_only_iterator& b)
+  {
+    return a.m_ptr != b.m_ptr;
+  }
+
+private:
+  pointer m_ptr;
+};
+
+_CCCL_HOST_DEVICE void TestOffsetIteratorBoth(thrust::offset_iterator<device_only_iterator> iter)
+{
+  assert(iter.offset() == 0);
+  ++iter;
+  assert(iter.offset() == 1);
+  iter++;
+  assert(iter.offset() == 2);
+  --iter;
+  assert(iter.offset() == 1);
+  iter--;
+  assert(iter.offset() == 0);
+  iter += 100;
+  assert(iter.offset() == 100);
+}
+
+__global__ void TestOffsetIteratorDevice(thrust::offset_iterator<device_only_iterator> iter)
+{
+  TestOffsetIteratorBoth(iter);
+
+  // access
+  assert(*iter == 1);
+
+  auto iter2 = iter;
+  iter2 += 3;
+  assert(*iter2 == 1);
+
+  // difference
+  assert(iter2 - iter == 3);
+
+  // comparison
+  assert(!(iter2 == iter));
+  assert(iter2 != iter);
+}
+
+void TestOffsetIteratorWithDeviceOnlyIterator()
+{
+  thrust::device_vector<int> v{1, 2, 3, 4, 5};
+  device_only_iterator base(thrust::raw_pointer_cast(v.data()));
+  thrust::offset_iterator iter(base);
+  TestOffsetIteratorBoth(iter);
+  TestOffsetIteratorDevice<<<1, 1>>>(iter);
+}
+DECLARE_UNITTEST(TestOffsetIteratorWithDeviceOnlyIterator);

cccl_upstream/thrust/testing/cuda/pair_sort.cu

@@ -0,0 +1,60 @@
+#include <thrust/execution_policy.h>
+#include <thrust/sort.h>
+
+#include <cuda/std/utility>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator>
+__global__ void stable_sort_kernel(ExecutionPolicy exec, Iterator first, Iterator last)
+{
+  thrust::stable_sort(exec, first, last);
+}
+
+struct make_pair_functor
+{
+  template <typename T1, typename T2>
+  _CCCL_HOST_DEVICE cuda::std::pair<T1, T2> operator()(const T1& x, const T2& y)
+  {
+    return cuda::std::make_pair(x, y);
+  } // end operator()()
+}; // end make_pair_functor
+
+template <typename ExecutionPolicy>
+void TestPairStableSortDevice(ExecutionPolicy exec)
+{
+  size_t n = 10000;
+  using P  = cuda::std::pair<int, int>;
+
+  thrust::host_vector<int> h_p1 = unittest::random_integers<int>(n);
+  thrust::host_vector<int> h_p2 = unittest::random_integers<int>(n);
+  thrust::host_vector<P> h_pairs(n);
+
+  // zip up pairs on the host
+  thrust::transform(h_p1.begin(), h_p1.end(), h_p2.begin(), h_pairs.begin(), make_pair_functor());
+
+  thrust::device_vector<P> d_pairs = h_pairs;
+
+  stable_sort_kernel<<<1, 1>>>(exec, d_pairs.begin(), d_pairs.end());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  // sort on the host
+  thrust::stable_sort(h_pairs.begin(), h_pairs.end());
+
+  ASSERT_EQUAL_QUIET(h_pairs, d_pairs);
+};
+
+void TestPairStableSortDeviceSeq()
+{
+  TestPairStableSortDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestPairStableSortDeviceSeq);
+
+void TestPairStableSortDeviceDevice()
+{
+  TestPairStableSortDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestPairStableSortDeviceDevice);
+#endif

cccl_upstream/thrust/testing/cuda/pair_sort_by_key.cu

@@ -0,0 +1,71 @@
+#include <thrust/execution_policy.h>
+#include <thrust/sequence.h>
+#include <thrust/sort.h>
+#include <thrust/transform.h>
+
+#include <cuda/std/utility>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
+__global__ void
+stable_sort_by_key_kernel(ExecutionPolicy exec, Iterator1 keys_first, Iterator1 keys_last, Iterator2 values_first)
+{
+  thrust::stable_sort_by_key(exec, keys_first, keys_last, values_first);
+}
+
+struct make_pair_functor
+{
+  template <typename T1, typename T2>
+  _CCCL_HOST_DEVICE cuda::std::pair<T1, T2> operator()(const T1& x, const T2& y)
+  {
+    return cuda::std::make_pair(x, y);
+  } // end operator()()
+}; // end make_pair_functor
+
+template <typename ExecutionPolicy>
+void TestPairStableSortByKeyDevice(ExecutionPolicy exec)
+{
+  size_t n = 10000;
+  using P  = cuda::std::pair<int, int>;
+
+  // host arrays
+  thrust::host_vector<int> h_p1 = unittest::random_integers<int>(n);
+  thrust::host_vector<int> h_p2 = unittest::random_integers<int>(n);
+  thrust::host_vector<P> h_pairs(n);
+
+  thrust::host_vector<int> h_values(n);
+  thrust::sequence(h_values.begin(), h_values.end());
+
+  // zip up pairs on the host
+  thrust::transform(h_p1.begin(), h_p1.end(), h_p2.begin(), h_pairs.begin(), make_pair_functor());
+
+  // device arrays
+  thrust::device_vector<P> d_pairs    = h_pairs;
+  thrust::device_vector<int> d_values = h_values;
+
+  // sort on the device
+  stable_sort_by_key_kernel<<<1, 1>>>(exec, d_pairs.begin(), d_pairs.end(), d_values.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  // sort on the host
+  thrust::stable_sort_by_key(h_pairs.begin(), h_pairs.end(), h_values.begin());
+
+  ASSERT_EQUAL_QUIET(h_pairs, d_pairs);
+  ASSERT_EQUAL(h_values, d_values);
+};
+
+void TestPairStableSortByKeyDeviceSeq()
+{
+  TestPairStableSortByKeyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestPairStableSortByKeyDeviceSeq);
+
+void TestPairStableSortByKeyDeviceDevice()
+{
+  TestPairStableSortByKeyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestPairStableSortByKeyDeviceDevice);
+#endif

cccl_upstream/thrust/testing/cuda/partition.cu

@@ -0,0 +1,734 @@
+#include <thrust/count.h>
+#include <thrust/execution_policy.h>
+#include <thrust/iterator/discard_iterator.h>
+#include <thrust/partition.h>
+
+#include "thrust/detail/raw_pointer_cast.h"
+#include <unittest/unittest.h>
+
+template <typename T>
+struct is_even
+{
+  _CCCL_HOST_DEVICE bool operator()(T x) const
+  {
+    return ((int) x % 2) == 0;
+  }
+};
+
+template <typename T>
+struct mod_n
+{
+  T mod;
+  bool negate;
+  _CCCL_HOST_DEVICE bool operator()(T x)
+  {
+    return (x % mod == 0) ? (!negate) : negate;
+  }
+};
+
+template <typename T>
+struct multiply_n
+{
+  T multiplier;
+  _CCCL_HOST_DEVICE T operator()(T x)
+  {
+    return x * multiplier;
+  }
+};
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Predicate, typename Iterator2>
+__global__ void partition_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Predicate pred, Iterator2 result)
+{
+  *result = thrust::partition(exec, first, last, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestPartitionDevice(ExecutionPolicy exec)
+{
+  using T        = int;
+  using iterator = typename thrust::device_vector<T>::iterator;
+
+  thrust::device_vector<T> data(5);
+  data[0] = 1;
+  data[1] = 2;
+  data[2] = 1;
+  data[3] = 1;
+  data[4] = 2;
+
+  thrust::device_vector<iterator> result(1);
+
+  partition_kernel<<<1, 1>>>(exec, data.begin(), data.end(), is_even<T>(), result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  thrust::device_vector<T> ref(5);
+  ref[0] = 2;
+  ref[1] = 2;
+  ref[2] = 1;
+  ref[3] = 1;
+  ref[4] = 1;
+
+  ASSERT_EQUAL(2, (iterator) result[0] - data.begin());
+  ASSERT_EQUAL(ref, data);
+}
+
+void TestPartitionDeviceSeq()
+{
+  TestPartitionDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestPartitionDeviceSeq);
+
+void TestPartitionDeviceDevice()
+{
+  TestPartitionDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestPartitionDeviceDevice);
+
+void TestPartitionDeviceNoSync()
+{
+  TestPartitionDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestPartitionDeviceNoSync);
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Predicate, typename Iterator3>
+__global__ void partition_kernel(
+  ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 stencil_first, Predicate pred, Iterator3 result)
+{
+  *result = thrust::partition(exec, first, last, stencil_first, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestPartitionStencilDevice(ExecutionPolicy exec)
+{
+  using T        = int;
+  using iterator = typename thrust::device_vector<T>::iterator;
+
+  thrust::device_vector<T> data(5);
+  data[0] = 0;
+  data[1] = 1;
+  data[2] = 0;
+  data[3] = 0;
+  data[4] = 1;
+
+  thrust::device_vector<T> stencil(5);
+  stencil[0] = 1;
+  stencil[1] = 2;
+  stencil[2] = 1;
+  stencil[3] = 1;
+  stencil[4] = 2;
+
+  thrust::device_vector<iterator> result(1);
+
+  partition_kernel<<<1, 1>>>(exec, data.begin(), data.end(), stencil.begin(), is_even<T>(), result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  thrust::device_vector<T> ref(5);
+  ref[0] = 1;
+  ref[1] = 1;
+  ref[2] = 0;
+  ref[3] = 0;
+  ref[4] = 0;
+
+  ASSERT_EQUAL(2, (iterator) result[0] - data.begin());
+  ASSERT_EQUAL(ref, data);
+}
+
+void TestPartitionStencilDeviceSeq()
+{
+  TestPartitionStencilDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestPartitionStencilDeviceSeq);
+
+void TestPartitionStencilDeviceDevice()
+{
+  TestPartitionStencilDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestPartitionStencilDeviceDevice);
+
+void TestPartitionStencilDeviceNoSync()
+{
+  TestPartitionStencilDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestPartitionStencilDeviceNoSync);
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Predicate,
+          typename Iterator4>
+__global__ void partition_copy_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first,
+  Iterator1 last,
+  Iterator2 true_result,
+  Iterator3 false_result,
+  Predicate pred,
+  Iterator4 result)
+{
+  *result = thrust::partition_copy(exec, first, last, true_result, false_result, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestPartitionCopyDevice(ExecutionPolicy exec)
+{
+  using T        = int;
+  using iterator = thrust::device_vector<T>::iterator;
+
+  thrust::device_vector<T> data(5);
+  data[0] = 1;
+  data[1] = 2;
+  data[2] = 1;
+  data[3] = 1;
+  data[4] = 2;
+
+  thrust::device_vector<int> true_results(2);
+  thrust::device_vector<int> false_results(3);
+
+  using pair_type = cuda::std::pair<iterator, iterator>;
+  thrust::device_vector<pair_type> iterators(1);
+
+  partition_copy_kernel<<<1, 1>>>(
+    exec, data.begin(), data.end(), true_results.begin(), false_results.begin(), is_even<T>(), iterators.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  thrust::device_vector<T> true_ref(2);
+  true_ref[0] = 2;
+  true_ref[1] = 2;
+
+  thrust::device_vector<T> false_ref(3);
+  false_ref[0] = 1;
+  false_ref[1] = 1;
+  false_ref[2] = 1;
+
+  pair_type ends = iterators[0];
+
+  ASSERT_EQUAL(2, ends.first - true_results.begin());
+  ASSERT_EQUAL(3, ends.second - false_results.begin());
+  ASSERT_EQUAL(true_ref, true_results);
+  ASSERT_EQUAL(false_ref, false_results);
+}
+
+void TestPartitionCopyDeviceSeq()
+{
+  TestPartitionCopyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestPartitionCopyDeviceSeq);
+
+void TestPartitionCopyDeviceDevice()
+{
+  TestPartitionCopyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestPartitionCopyDeviceDevice);
+
+void TestPartitionCopyDeviceNoSync()
+{
+  TestPartitionCopyDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestPartitionCopyDeviceNoSync);
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Iterator4,
+          typename Predicate,
+          typename Iterator5>
+__global__ void partition_copy_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first,
+  Iterator1 last,
+  Iterator2 stencil_first,
+  Iterator3 true_result,
+  Iterator4 false_result,
+  Predicate pred,
+  Iterator5 result)
+{
+  *result = thrust::partition_copy(exec, first, last, stencil_first, true_result, false_result, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestPartitionCopyStencilDevice(ExecutionPolicy exec)
+{
+  using T = int;
+
+  thrust::device_vector<int> data(5);
+  data[0] = 0;
+  data[1] = 1;
+  data[2] = 0;
+  data[3] = 0;
+  data[4] = 1;
+
+  thrust::device_vector<int> stencil(5);
+  stencil[0] = 1;
+  stencil[1] = 2;
+  stencil[2] = 1;
+  stencil[3] = 1;
+  stencil[4] = 2;
+
+  thrust::device_vector<int> true_results(2);
+  thrust::device_vector<int> false_results(3);
+
+  using iterator  = typename thrust::device_vector<int>::iterator;
+  using pair_type = cuda::std::pair<iterator, iterator>;
+  thrust::device_vector<pair_type> iterators(1);
+
+  partition_copy_kernel<<<1, 1>>>(
+    exec,
+    data.begin(),
+    data.end(),
+    stencil.begin(),
+    true_results.begin(),
+    false_results.begin(),
+    is_even<T>(),
+    iterators.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  pair_type ends = iterators[0];
+
+  thrust::device_vector<int> true_ref(2);
+  true_ref[0] = 1;
+  true_ref[1] = 1;
+
+  thrust::device_vector<int> false_ref(3);
+  false_ref[0] = 0;
+  false_ref[1] = 0;
+  false_ref[2] = 0;
+
+  ASSERT_EQUAL(2, ends.first - true_results.begin());
+  ASSERT_EQUAL(3, ends.second - false_results.begin());
+  ASSERT_EQUAL(true_ref, true_results);
+  ASSERT_EQUAL(false_ref, false_results);
+}
+
+void TestPartitionCopyStencilDeviceSeq()
+{
+  TestPartitionCopyStencilDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestPartitionCopyStencilDeviceSeq);
+
+void TestPartitionCopyStencilDeviceDevice()
+{
+  TestPartitionCopyStencilDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestPartitionCopyStencilDeviceDevice);
+
+void TestPartitionCopyStencilDeviceNoSync()
+{
+  TestPartitionCopyStencilDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestPartitionCopyStencilDeviceNoSync);
+
+template <typename ExecutionPolicy, typename Iterator1, typename Predicate, typename Iterator2>
+__global__ void
+stable_partition_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Predicate pred, Iterator2 result)
+{
+  *result = thrust::stable_partition(exec, first, last, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestStablePartitionDevice(ExecutionPolicy exec)
+{
+  using T        = int;
+  using iterator = typename thrust::device_vector<T>::iterator;
+
+  thrust::device_vector<T> data(5);
+  data[0] = 1;
+  data[1] = 2;
+  data[2] = 1;
+  data[3] = 1;
+  data[4] = 2;
+
+  thrust::device_vector<iterator> result(1);
+
+  stable_partition_kernel<<<1, 1>>>(exec, data.begin(), data.end(), is_even<T>(), result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  thrust::device_vector<T> ref(5);
+  ref[0] = 2;
+  ref[1] = 2;
+  ref[2] = 1;
+  ref[3] = 1;
+  ref[4] = 1;
+
+  ASSERT_EQUAL(2, (iterator) result[0] - data.begin());
+  ASSERT_EQUAL(ref, data);
+}
+
+void TestStablePartitionDeviceSeq()
+{
+  TestStablePartitionDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestStablePartitionDeviceSeq);
+
+void TestStablePartitionDeviceDevice()
+{
+  TestStablePartitionDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestStablePartitionDeviceDevice);
+
+void TestStablePartitionDeviceNoSync()
+{
+  TestStablePartitionDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestStablePartitionDeviceNoSync);
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Predicate, typename Iterator3>
+__global__ void stable_partition_kernel(
+  ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 stencil_first, Predicate pred, Iterator3 result)
+{
+  *result = thrust::stable_partition(exec, first, last, stencil_first, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestStablePartitionStencilDevice(ExecutionPolicy exec)
+{
+  using T        = int;
+  using iterator = typename thrust::device_vector<T>::iterator;
+
+  thrust::device_vector<T> data(5);
+  data[0] = 0;
+  data[1] = 1;
+  data[2] = 0;
+  data[3] = 0;
+  data[4] = 1;
+
+  thrust::device_vector<T> stencil(5);
+  stencil[0] = 1;
+  stencil[1] = 2;
+  stencil[2] = 1;
+  stencil[3] = 1;
+  stencil[4] = 2;
+
+  thrust::device_vector<iterator> result(1);
+
+  stable_partition_kernel<<<1, 1>>>(exec, data.begin(), data.end(), stencil.begin(), is_even<T>(), result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  thrust::device_vector<T> ref(5);
+  ref[0] = 1;
+  ref[1] = 1;
+  ref[2] = 0;
+  ref[3] = 0;
+  ref[4] = 0;
+
+  ASSERT_EQUAL(2, (iterator) result[0] - data.begin());
+  ASSERT_EQUAL(ref, data);
+}
+
+void TestStablePartitionStencilDeviceSeq()
+{
+  TestStablePartitionStencilDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestStablePartitionStencilDeviceSeq);
+
+void TestStablePartitionStencilDeviceDevice()
+{
+  TestStablePartitionStencilDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestStablePartitionStencilDeviceDevice);
+
+void TestStablePartitionStencilDeviceNoSync()
+{
+  TestStablePartitionStencilDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestStablePartitionStencilDeviceNoSync);
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Predicate,
+          typename Iterator4>
+__global__ void stable_partition_copy_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first,
+  Iterator1 last,
+  Iterator2 true_result,
+  Iterator3 false_result,
+  Predicate pred,
+  Iterator4 result)
+{
+  *result = thrust::stable_partition_copy(exec, first, last, true_result, false_result, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestStablePartitionCopyDevice(ExecutionPolicy exec)
+{
+  using T        = int;
+  using iterator = thrust::device_vector<T>::iterator;
+
+  thrust::device_vector<T> data(5);
+  data[0] = 1;
+  data[1] = 2;
+  data[2] = 1;
+  data[3] = 1;
+  data[4] = 2;
+
+  thrust::device_vector<int> true_results(2);
+  thrust::device_vector<int> false_results(3);
+
+  using pair_type = cuda::std::pair<iterator, iterator>;
+  thrust::device_vector<pair_type> iterators(1);
+
+  stable_partition_copy_kernel<<<1, 1>>>(
+    exec, data.begin(), data.end(), true_results.begin(), false_results.begin(), is_even<T>(), iterators.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  thrust::device_vector<T> true_ref(2);
+  true_ref[0] = 2;
+  true_ref[1] = 2;
+
+  thrust::device_vector<T> false_ref(3);
+  false_ref[0] = 1;
+  false_ref[1] = 1;
+  false_ref[2] = 1;
+
+  pair_type ends = iterators[0];
+
+  ASSERT_EQUAL(2, ends.first - true_results.begin());
+  ASSERT_EQUAL(3, ends.second - false_results.begin());
+  ASSERT_EQUAL(true_ref, true_results);
+  ASSERT_EQUAL(false_ref, false_results);
+}
+
+void TestStablePartitionCopyDeviceSeq()
+{
+  TestStablePartitionCopyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestStablePartitionCopyDeviceSeq);
+
+void TestStablePartitionCopyDeviceDevice()
+{
+  TestStablePartitionCopyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestStablePartitionCopyDeviceDevice);
+
+void TestStablePartitionCopyDeviceNoSync()
+{
+  TestStablePartitionCopyDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestStablePartitionCopyDeviceNoSync);
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Iterator4,
+          typename Predicate,
+          typename Iterator5>
+__global__ void stable_partition_copy_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first,
+  Iterator1 last,
+  Iterator2 stencil_first,
+  Iterator3 true_result,
+  Iterator4 false_result,
+  Predicate pred,
+  Iterator5 result)
+{
+  *result = thrust::stable_partition_copy(exec, first, last, stencil_first, true_result, false_result, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestStablePartitionCopyStencilDevice(ExecutionPolicy exec)
+{
+  using T = int;
+
+  thrust::device_vector<int> data(5);
+  data[0] = 0;
+  data[1] = 1;
+  data[2] = 0;
+  data[3] = 0;
+  data[4] = 1;
+
+  thrust::device_vector<int> stencil(5);
+  stencil[0] = 1;
+  stencil[1] = 2;
+  stencil[2] = 1;
+  stencil[3] = 1;
+  stencil[4] = 2;
+
+  thrust::device_vector<int> true_results(2);
+  thrust::device_vector<int> false_results(3);
+
+  using iterator  = typename thrust::device_vector<int>::iterator;
+  using pair_type = cuda::std::pair<iterator, iterator>;
+  thrust::device_vector<pair_type> iterators(1);
+
+  stable_partition_copy_kernel<<<1, 1>>>(
+    exec,
+    data.begin(),
+    data.end(),
+    stencil.begin(),
+    true_results.begin(),
+    false_results.begin(),
+    is_even<T>(),
+    iterators.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  pair_type ends = iterators[0];
+
+  thrust::device_vector<int> true_ref(2);
+  true_ref[0] = 1;
+  true_ref[1] = 1;
+
+  thrust::device_vector<int> false_ref(3);
+  false_ref[0] = 0;
+  false_ref[1] = 0;
+  false_ref[2] = 0;
+
+  ASSERT_EQUAL(2, ends.first - true_results.begin());
+  ASSERT_EQUAL(3, ends.second - false_results.begin());
+  ASSERT_EQUAL(true_ref, true_results);
+  ASSERT_EQUAL(false_ref, false_results);
+}
+
+void TestStablePartitionCopyStencilDeviceSeq()
+{
+  TestStablePartitionCopyStencilDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestStablePartitionCopyStencilDeviceSeq);
+
+void TestStablePartitionCopyStencilDeviceDevice()
+{
+  TestStablePartitionCopyStencilDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestStablePartitionCopyStencilDeviceDevice);
+
+void TestStablePartitionCopyStencilDeviceNoSync()
+{
+  TestStablePartitionCopyStencilDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestStablePartitionCopyStencilDeviceNoSync);
+
+void TestPartitionIfWithMagnitude(int magnitude)
+{
+  using offset_t = std::size_t;
+
+  // Prepare input
+  offset_t num_items = offset_t{1ull} << magnitude;
+  thrust::counting_iterator<offset_t> begin(offset_t{0});
+  auto end = begin + num_items;
+  thrust::counting_iterator<offset_t> stencil(offset_t{0});
+  ASSERT_EQUAL(static_cast<offset_t>(::cuda::std::distance(begin, end)), num_items);
+
+  // Run algorithm on large number of items
+  offset_t match_every_nth      = 1000000;
+  offset_t expected_num_written = (num_items + match_every_nth - 1) / match_every_nth;
+
+  // Tests input is correctly dereferenced for large offsets and selected items are correctly written
+  {
+    // Initialize input
+    thrust::device_vector<offset_t> partitioned_out(expected_num_written);
+
+    // Run test
+    constexpr bool negate_matches = false;
+    auto select_op                = mod_n<offset_t>{match_every_nth, negate_matches};
+    auto partitioned_out_ends =
+      thrust::stable_partition_copy(begin, end, partitioned_out.begin(), thrust::make_discard_iterator(), select_op);
+    const auto selected_out_end = partitioned_out_ends.first;
+
+    // Ensure number of selected items are correct
+    const offset_t num_selected_out =
+      static_cast<offset_t>(::cuda::std::distance(partitioned_out.begin(), selected_out_end));
+    ASSERT_EQUAL(num_selected_out, expected_num_written);
+    partitioned_out.resize(expected_num_written);
+
+    // Ensure selected items are correct
+    auto expected_out_it     = thrust::make_transform_iterator(begin, multiply_n<offset_t>{match_every_nth});
+    bool all_results_correct = thrust::equal(partitioned_out.begin(), partitioned_out.end(), expected_out_it);
+    ASSERT_EQUAL(all_results_correct, true);
+  }
+
+  // Tests input is correctly dereferenced for large offsets and rejected items are correctly written
+  {
+    // Initialize input
+    thrust::device_vector<offset_t> partitioned_out(expected_num_written);
+
+    // Run test
+    constexpr bool negate_matches = true;
+    auto select_op                = mod_n<offset_t>{match_every_nth, negate_matches};
+    const auto partitioned_out_ends =
+      thrust::stable_partition_copy(begin, end, thrust::make_discard_iterator(), partitioned_out.begin(), select_op);
+    const auto rejected_out_end = partitioned_out_ends.second;
+
+    // Ensure number of rejected items are correct
+    const offset_t num_rejected_out =
+      static_cast<offset_t>(::cuda::std::distance(partitioned_out.begin(), rejected_out_end));
+    ASSERT_EQUAL(num_rejected_out, expected_num_written);
+    partitioned_out.resize(expected_num_written);
+
+    // Ensure rejected items are correct
+    auto expected_out_it     = thrust::make_transform_iterator(begin, multiply_n<offset_t>{match_every_nth});
+    bool all_results_correct = thrust::equal(partitioned_out.begin(), partitioned_out.end(), expected_out_it);
+    ASSERT_EQUAL(all_results_correct, true);
+  }
+}
+
+void TestPartitionIfWithLargeNumberOfItems()
+{
+  TestPartitionIfWithMagnitude(30);
+  // These require 64-bit dispatches even when magnitude < 32.
+#  ifndef THRUST_FORCE_32_BIT_OFFSET_TYPE
+  TestPartitionIfWithMagnitude(31);
+  TestPartitionIfWithMagnitude(32);
+  TestPartitionIfWithMagnitude(33);
+#  endif
+}
+DECLARE_UNITTEST(TestPartitionIfWithLargeNumberOfItems);
+#endif
+
+template <typename ExecutionPolicy>
+void TestPartitionCudaStreams(ExecutionPolicy policy)
+{
+  using Vector   = thrust::device_vector<int>;
+  using T        = Vector::value_type;
+  using Iterator = Vector::iterator;
+
+  Vector data(5);
+  data[0] = 1;
+  data[1] = 2;
+  data[2] = 1;
+  data[3] = 1;
+  data[4] = 2;
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  auto streampolicy = policy.on(s);
+
+  Iterator iter = thrust::partition(streampolicy, data.begin(), data.end(), is_even<T>());
+
+  Vector ref(5);
+  ref[0] = 2;
+  ref[1] = 2;
+  ref[2] = 1;
+  ref[3] = 1;
+  ref[4] = 1;
+
+  ASSERT_EQUAL(iter - data.begin(), 2);
+  ASSERT_EQUAL(data, ref);
+
+  cudaStreamDestroy(s);
+}
+
+void TestPartitionCudaStreamsSync()
+{
+  TestPartitionCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestPartitionCudaStreamsSync);
+
+void TestPartitionCudaStreamsNoSync()
+{
+  TestPartitionCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestPartitionCudaStreamsNoSync);

cccl_upstream/thrust/testing/cuda/partition_point.cu

@@ -0,0 +1,82 @@
+#include <thrust/execution_policy.h>
+#include <thrust/functional.h>
+#include <thrust/partition.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Predicate, typename Iterator2>
+__global__ void
+partition_point_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Predicate pred, Iterator2 result)
+{
+  *result = thrust::partition_point(exec, first, last, pred);
+}
+
+template <typename T>
+struct is_even
+{
+  _CCCL_HOST_DEVICE bool operator()(T x) const
+  {
+    return ((int) x % 2) == 0;
+  }
+};
+
+template <typename ExecutionPolicy>
+void TestPartitionPointDevice(ExecutionPolicy exec)
+{
+  size_t n                     = 1000;
+  thrust::device_vector<int> v = unittest::random_integers<int>(n);
+  using iterator               = typename thrust::device_vector<int>::iterator;
+
+  iterator ref = thrust::stable_partition(v.begin(), v.end(), is_even<int>());
+
+  thrust::device_vector<iterator> result(1);
+  partition_point_kernel<<<1, 1>>>(exec, v.begin(), v.end(), is_even<int>(), result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(ref - v.begin(), (iterator) result[0] - v.begin());
+}
+
+void TestPartitionPointDeviceSeq()
+{
+  TestPartitionPointDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestPartitionPointDeviceSeq);
+
+void TestPartitionPointDeviceDevice()
+{
+  TestPartitionPointDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestPartitionPointDeviceDevice);
+#endif
+
+void TestPartitionPointCudaStreams()
+{
+  using Vector   = thrust::device_vector<int>;
+  using T        = Vector::value_type;
+  using Iterator = Vector::iterator;
+
+  Vector v(4);
+  v[0] = 1;
+  v[1] = 1;
+  v[2] = 1;
+  v[3] = 0;
+
+  Iterator first = v.begin();
+
+  Iterator last = v.begin() + 4;
+  Iterator ref  = first + 3;
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  ASSERT_EQUAL_QUIET(ref, thrust::partition_point(thrust::cuda::par.on(s), first, last, ::cuda::std::identity{}));
+
+  last = v.begin() + 3;
+  ref  = last;
+  ASSERT_EQUAL_QUIET(ref, thrust::partition_point(thrust::cuda::par.on(s), first, last, ::cuda::std::identity{}));
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestPartitionPointCudaStreams);

cccl_upstream/thrust/testing/cuda/reduce.cu

@@ -0,0 +1,118 @@
+#include <thrust/execution_policy.h>
+#include <thrust/reduce.h>
+
+#include <cuda/iterator>
+
+#include <unittest/unittest.h>
+
+template <typename ExecutionPolicy, typename Iterator, typename T, typename Iterator2>
+__global__ void reduce_kernel(ExecutionPolicy exec, Iterator first, Iterator last, T init, Iterator2 result)
+{
+  *result = thrust::reduce(exec, first, last, init);
+}
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename T, typename ExecutionPolicy>
+void TestReduceDevice(ExecutionPolicy exec, const size_t n)
+{
+  thrust::host_vector<T> h_data   = unittest::random_integers<T>(n);
+  thrust::device_vector<T> d_data = h_data;
+
+  thrust::device_vector<T> d_result(1);
+
+  T init = 13;
+
+  T h_result = thrust::reduce(h_data.begin(), h_data.end(), init);
+
+  reduce_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), init, d_result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(h_result, d_result[0]);
+}
+
+template <typename T>
+struct TestReduceDeviceSeq
+{
+  void operator()(const size_t n)
+  {
+    TestReduceDevice<T>(thrust::seq, n);
+  }
+};
+VariableUnitTest<TestReduceDeviceSeq, IntegralTypes> TestReduceDeviceSeqInstance;
+
+template <typename T>
+struct TestReduceDeviceDevice
+{
+  void operator()(const size_t n)
+  {
+    TestReduceDevice<T>(thrust::device, n);
+  }
+};
+VariableUnitTest<TestReduceDeviceDevice, IntegralTypes> TestReduceDeviceDeviceInstance;
+
+template <typename T>
+struct TestReduceDeviceNoSync
+{
+  void operator()(const size_t n)
+  {
+    TestReduceDevice<T>(thrust::cuda::par_nosync, n);
+  }
+};
+VariableUnitTest<TestReduceDeviceNoSync, IntegralTypes> TestReduceDeviceNoSyncInstance;
+#endif
+
+template <typename ExecutionPolicy>
+void TestReduceCudaStreams(ExecutionPolicy policy)
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector v(3);
+  v[0] = 1;
+  v[1] = -2;
+  v[2] = 3;
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  auto streampolicy = policy.on(s);
+
+  // no initializer
+  ASSERT_EQUAL(thrust::reduce(streampolicy, v.begin(), v.end()), 2);
+
+  // with initializer
+  ASSERT_EQUAL(thrust::reduce(streampolicy, v.begin(), v.end(), 10), 12);
+
+  cudaStreamDestroy(s);
+}
+
+void TestReduceCudaStreamsSync()
+{
+  TestReduceCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestReduceCudaStreamsSync);
+
+void TestReduceCudaStreamsNoSync()
+{
+  TestReduceCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestReduceCudaStreamsNoSync);
+
+#if defined(THRUST_RDC_ENABLED)
+void TestReduceLargeInput()
+{
+  using T                 = unsigned long long;
+  using OffsetT           = std::size_t;
+  const OffsetT num_items = 1ull << 32;
+
+  cuda::constant_iterator<T> d_data(T{1});
+  thrust::device_vector<T> d_result(1);
+
+  reduce_kernel<<<1, 1>>>(thrust::device, d_data, d_data + num_items, T{}, d_result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(num_items, d_result[0]);
+}
+DECLARE_UNITTEST(TestReduceLargeInput);
+#endif

cccl_upstream/thrust/testing/cuda/reduce_by_key.cu

@@ -0,0 +1,577 @@
+#include <thrust/device_vector.h>
+#include <thrust/equal.h>
+#include <thrust/execution_policy.h>
+#include <thrust/iterator/transform_iterator.h>
+#include <thrust/reduce.h>
+
+#include <cuda/iterator>
+#include <cuda/std/functional>
+
+#include <cstdint>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Iterator4,
+          typename Iterator5>
+__global__ void reduce_by_key_kernel(
+  ExecutionPolicy exec,
+  Iterator1 keys_first,
+  Iterator1 keys_last,
+  Iterator2 values_first,
+  Iterator3 keys_result,
+  Iterator4 values_result,
+  Iterator5 result)
+{
+  *result = thrust::reduce_by_key(exec, keys_first, keys_last, values_first, keys_result, values_result);
+}
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Iterator4,
+          typename BinaryPredicate,
+          typename Iterator5>
+__global__ void reduce_by_key_kernel(
+  ExecutionPolicy exec,
+  Iterator1 keys_first,
+  Iterator1 keys_last,
+  Iterator2 values_first,
+  Iterator3 keys_result,
+  Iterator4 values_result,
+  BinaryPredicate pred,
+  Iterator5 result)
+{
+  *result = thrust::reduce_by_key(exec, keys_first, keys_last, values_first, keys_result, values_result, pred);
+}
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Iterator4,
+          typename BinaryPredicate,
+          typename BinaryFunction,
+          typename Iterator5>
+__global__ void reduce_by_key_kernel(
+  ExecutionPolicy exec,
+  Iterator1 keys_first,
+  Iterator1 keys_last,
+  Iterator2 values_first,
+  Iterator3 keys_result,
+  Iterator4 values_result,
+  BinaryPredicate pred,
+  BinaryFunction binary_op,
+  Iterator5 result)
+{
+  *result =
+    thrust::reduce_by_key(exec, keys_first, keys_last, values_first, keys_result, values_result, pred, binary_op);
+}
+#endif
+
+template <typename T>
+struct is_equal_div_10_reduce
+{
+  _CCCL_HOST_DEVICE bool operator()(const T x, const T& y) const
+  {
+    return ((int) x / 10) == ((int) y / 10);
+  }
+};
+
+template <typename Vector>
+void initialize_keys(Vector& keys)
+{
+  keys.resize(9);
+  keys[0] = 11;
+  keys[1] = 11;
+  keys[2] = 21;
+  keys[3] = 20;
+  keys[4] = 21;
+  keys[5] = 21;
+  keys[6] = 21;
+  keys[7] = 37;
+  keys[8] = 37;
+}
+
+template <typename Vector>
+void initialize_values(Vector& values)
+{
+  values.resize(9);
+  values[0] = 0;
+  values[1] = 1;
+  values[2] = 2;
+  values[3] = 3;
+  values[4] = 4;
+  values[5] = 5;
+  values[6] = 6;
+  values[7] = 7;
+  values[8] = 8;
+}
+
+// Checks whether the equality operator is ever invoked on out-of-bounds items
+struct check_valid_item_op
+{
+  cuda::std::uint32_t* error_counter{};
+  int expected_upper_bound{};
+
+  __device__ bool operator()(const int lhs, const int rhs) const
+  {
+    if (lhs > expected_upper_bound || rhs > expected_upper_bound)
+    {
+      if (error_counter)
+      {
+        atomicAdd(error_counter, 1);
+      }
+      return false;
+    }
+    return lhs == rhs;
+  }
+};
+
+template <typename ReturnedT>
+struct check_accumulator_t_op
+{
+  cuda::std::uint32_t* error_counter{};
+
+  template <typename T,
+            typename U,
+            typename = cuda::std::enable_if_t<cuda::std::is_same_v<T, U> && !cuda::std::is_same_v<T, ReturnedT>>>
+  _CCCL_DEVICE ReturnedT operator()(const T lhs, const U& rhs) const
+  {
+    return static_cast<ReturnedT>(lhs + rhs);
+  }
+
+  template <typename T>
+  _CCCL_DEVICE ReturnedT operator()(const ReturnedT lhs, const T& rhs) const
+  {
+    atomicAdd(error_counter, 1);
+    return lhs + static_cast<ReturnedT>(rhs);
+  }
+};
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy>
+void TestReduceByKeyDevice(ExecutionPolicy exec)
+{
+  using T = int;
+
+  thrust::device_vector<T> keys;
+  thrust::device_vector<T> values;
+
+  using iterator_pair =
+    typename cuda::std::pair<typename thrust::device_vector<T>::iterator, typename thrust::device_vector<T>::iterator>;
+
+  thrust::device_vector<iterator_pair> new_last_vec(1);
+  iterator_pair new_last;
+
+  // basic test
+  initialize_keys(keys);
+  initialize_values(values);
+
+  thrust::device_vector<T> output_keys(keys.size());
+  thrust::device_vector<T> output_values(values.size());
+
+  reduce_by_key_kernel<<<1, 1>>>(
+    exec, keys.begin(), keys.end(), values.begin(), output_keys.begin(), output_values.begin(), new_last_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  new_last = new_last_vec[0];
+
+  ASSERT_EQUAL(new_last.first - output_keys.begin(), 5);
+  ASSERT_EQUAL(new_last.second - output_values.begin(), 5);
+  ASSERT_EQUAL(output_keys[0], 11);
+  ASSERT_EQUAL(output_keys[1], 21);
+  ASSERT_EQUAL(output_keys[2], 20);
+  ASSERT_EQUAL(output_keys[3], 21);
+  ASSERT_EQUAL(output_keys[4], 37);
+
+  ASSERT_EQUAL(output_values[0], 1);
+  ASSERT_EQUAL(output_values[1], 2);
+  ASSERT_EQUAL(output_values[2], 3);
+  ASSERT_EQUAL(output_values[3], 15);
+  ASSERT_EQUAL(output_values[4], 15);
+
+  // test BinaryPredicate
+  initialize_keys(keys);
+  initialize_values(values);
+
+  reduce_by_key_kernel<<<1, 1>>>(
+    exec,
+    keys.begin(),
+    keys.end(),
+    values.begin(),
+    output_keys.begin(),
+    output_values.begin(),
+    is_equal_div_10_reduce<T>(),
+    new_last_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  new_last = new_last_vec[0];
+
+  ASSERT_EQUAL(new_last.first - output_keys.begin(), 3);
+  ASSERT_EQUAL(new_last.second - output_values.begin(), 3);
+  ASSERT_EQUAL(output_keys[0], 11);
+  ASSERT_EQUAL(output_keys[1], 21);
+  ASSERT_EQUAL(output_keys[2], 37);
+
+  ASSERT_EQUAL(output_values[0], 1);
+  ASSERT_EQUAL(output_values[1], 20);
+  ASSERT_EQUAL(output_values[2], 15);
+
+  // test BinaryFunction
+  initialize_keys(keys);
+  initialize_values(values);
+
+  reduce_by_key_kernel<<<1, 1>>>(
+    exec,
+    keys.begin(),
+    keys.end(),
+    values.begin(),
+    output_keys.begin(),
+    output_values.begin(),
+    ::cuda::std::equal_to<T>(),
+    ::cuda::std::plus<T>(),
+    new_last_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  new_last = new_last_vec[0];
+
+  ASSERT_EQUAL(new_last.first - output_keys.begin(), 5);
+  ASSERT_EQUAL(new_last.second - output_values.begin(), 5);
+  ASSERT_EQUAL(output_keys[0], 11);
+  ASSERT_EQUAL(output_keys[1], 21);
+  ASSERT_EQUAL(output_keys[2], 20);
+  ASSERT_EQUAL(output_keys[3], 21);
+  ASSERT_EQUAL(output_keys[4], 37);
+
+  ASSERT_EQUAL(output_values[0], 1);
+  ASSERT_EQUAL(output_values[1], 2);
+  ASSERT_EQUAL(output_values[2], 3);
+  ASSERT_EQUAL(output_values[3], 15);
+  ASSERT_EQUAL(output_values[4], 15);
+}
+
+void TestReduceByKeyDeviceSeq()
+{
+  TestReduceByKeyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestReduceByKeyDeviceSeq);
+
+void TestReduceByKeyDeviceDevice()
+{
+  TestReduceByKeyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestReduceByKeyDeviceDevice);
+
+void TestReduceByKeyDeviceNoSync()
+{
+  TestReduceByKeyDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestReduceByKeyDeviceNoSync);
+#endif
+
+template <typename ExecutionPolicy>
+void TestReduceByKeyCudaStreams(ExecutionPolicy policy)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector keys;
+  Vector values;
+
+  cuda::std::pair<Vector::iterator, Vector::iterator> new_last;
+
+  // basic test
+  initialize_keys(keys);
+  initialize_values(values);
+
+  Vector output_keys(keys.size());
+  Vector output_values(values.size());
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  auto streampolicy = policy.on(s);
+
+  new_last = thrust::reduce_by_key(
+    streampolicy, keys.begin(), keys.end(), values.begin(), output_keys.begin(), output_values.begin());
+
+  ASSERT_EQUAL(new_last.first - output_keys.begin(), 5);
+  ASSERT_EQUAL(new_last.second - output_values.begin(), 5);
+  ASSERT_EQUAL(output_keys[0], 11);
+  ASSERT_EQUAL(output_keys[1], 21);
+  ASSERT_EQUAL(output_keys[2], 20);
+  ASSERT_EQUAL(output_keys[3], 21);
+  ASSERT_EQUAL(output_keys[4], 37);
+
+  ASSERT_EQUAL(output_values[0], 1);
+  ASSERT_EQUAL(output_values[1], 2);
+  ASSERT_EQUAL(output_values[2], 3);
+  ASSERT_EQUAL(output_values[3], 15);
+  ASSERT_EQUAL(output_values[4], 15);
+
+  // test BinaryPredicate
+  initialize_keys(keys);
+  initialize_values(values);
+
+  new_last = thrust::reduce_by_key(
+    streampolicy,
+    keys.begin(),
+    keys.end(),
+    values.begin(),
+    output_keys.begin(),
+    output_values.begin(),
+    is_equal_div_10_reduce<T>());
+
+  ASSERT_EQUAL(new_last.first - output_keys.begin(), 3);
+  ASSERT_EQUAL(new_last.second - output_values.begin(), 3);
+  ASSERT_EQUAL(output_keys[0], 11);
+  ASSERT_EQUAL(output_keys[1], 21);
+  ASSERT_EQUAL(output_keys[2], 37);
+
+  ASSERT_EQUAL(output_values[0], 1);
+  ASSERT_EQUAL(output_values[1], 20);
+  ASSERT_EQUAL(output_values[2], 15);
+
+  // test BinaryFunction
+  initialize_keys(keys);
+  initialize_values(values);
+
+  new_last = thrust::reduce_by_key(
+    streampolicy,
+    keys.begin(),
+    keys.end(),
+    values.begin(),
+    output_keys.begin(),
+    output_values.begin(),
+    ::cuda::std::equal_to<T>(),
+    ::cuda::std::plus<T>());
+
+  ASSERT_EQUAL(new_last.first - output_keys.begin(), 5);
+  ASSERT_EQUAL(new_last.second - output_values.begin(), 5);
+  ASSERT_EQUAL(output_keys[0], 11);
+  ASSERT_EQUAL(output_keys[1], 21);
+  ASSERT_EQUAL(output_keys[2], 20);
+  ASSERT_EQUAL(output_keys[3], 21);
+  ASSERT_EQUAL(output_keys[4], 37);
+
+  ASSERT_EQUAL(output_values[0], 1);
+  ASSERT_EQUAL(output_values[1], 2);
+  ASSERT_EQUAL(output_values[2], 3);
+  ASSERT_EQUAL(output_values[3], 15);
+  ASSERT_EQUAL(output_values[4], 15);
+
+  cudaStreamDestroy(s);
+}
+
+void TestReduceByKeyCudaStreamsSync()
+{
+  TestReduceByKeyCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestReduceByKeyCudaStreamsSync);
+
+void TestReduceByKeyCudaStreamsNoSync()
+{
+  TestReduceByKeyCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestReduceByKeyCudaStreamsNoSync);
+
+// Maps indices to key ids
+class div_op
+{
+  std::int64_t m_divisor;
+
+public:
+  _CCCL_HOST div_op(std::int64_t divisor)
+      : m_divisor(divisor)
+  {}
+
+  _CCCL_HOST_DEVICE std::int64_t operator()(std::int64_t x) const
+  {
+    return x / m_divisor;
+  }
+};
+
+// Produces unique sequence for key
+class mod_op
+{
+  std::int64_t m_divisor;
+
+public:
+  _CCCL_HOST mod_op(std::int64_t divisor)
+      : m_divisor(divisor)
+  {}
+
+  _CCCL_HOST_DEVICE std::int64_t operator()(std::int64_t x) const
+  {
+    // div: 2
+    // idx: 0 1   2 3   4 5
+    // key: 0 0 | 1 1 | 2 2
+    // mod: 0 1 | 0 1 | 0 1
+    // ret: 0 1   1 2   2 3
+    return (x % m_divisor) + (x / m_divisor);
+  }
+};
+
+void TestReduceByKeyWithBigIndexesHelper(int magnitude)
+{
+  const std::int64_t key_size_magnitude = 8;
+  ASSERT_EQUAL(true, key_size_magnitude < magnitude);
+
+  const std::int64_t num_items       = 1ll << magnitude;
+  const std::int64_t num_unique_keys = 1ll << key_size_magnitude;
+
+  // Size of each key group
+  const std::int64_t key_size = num_items / num_unique_keys;
+
+  using counting_it      = thrust::counting_iterator<std::int64_t>;
+  using transform_key_it = thrust::transform_iterator<div_op, counting_it>;
+  using transform_val_it = thrust::transform_iterator<mod_op, counting_it>;
+
+  counting_it count_begin(0ll);
+  counting_it count_end = count_begin + num_items;
+  ASSERT_EQUAL(static_cast<std::int64_t>(::cuda::std::distance(count_begin, count_end)), num_items);
+
+  transform_key_it keys_begin(count_begin, div_op{key_size});
+  transform_key_it keys_end(count_end, div_op{key_size});
+
+  transform_val_it values_begin(count_begin, mod_op{key_size});
+
+  thrust::device_vector<std::int64_t> output_keys(num_unique_keys);
+  thrust::device_vector<std::int64_t> output_values(num_unique_keys);
+
+  // example:
+  //  items:        6
+  //  unique_keys:  2
+  //  key_size:     3
+  //  keys:         0 0 0 | 1 1 1
+  //  values:       0 1 2 | 1 2 3
+  //  result:       3       6     = sum(range(key_size)) + key_size * key_id
+  thrust::reduce_by_key(keys_begin, keys_end, values_begin, output_keys.begin(), output_values.begin());
+
+  ASSERT_EQUAL(true, thrust::equal(output_keys.begin(), output_keys.end(), count_begin));
+
+  thrust::host_vector<std::int64_t> result = output_values;
+
+  const std::int64_t sum = (key_size - 1) * key_size / 2;
+  for (std::int64_t key_id = 0; key_id < num_unique_keys; key_id++)
+  {
+    ASSERT_EQUAL(result[key_id], sum + key_id * key_size);
+  }
+}
+
+void TestReduceByKeyWithBigIndexes()
+{
+  TestReduceByKeyWithBigIndexesHelper(30);
+#ifndef THRUST_FORCE_32_BIT_OFFSET_TYPE
+  TestReduceByKeyWithBigIndexesHelper(31);
+  TestReduceByKeyWithBigIndexesHelper(32);
+  TestReduceByKeyWithBigIndexesHelper(33);
+#endif
+}
+DECLARE_UNITTEST(TestReduceByKeyWithBigIndexes);
+
+void TestReduceByKeyWithCustomEqualityOp()
+{
+  using key_vector_t = thrust::device_vector<cuda::std::int32_t>;
+  using val_vector_t = thrust::device_vector<cuda::std::int32_t>;
+  using key_t        = key_vector_t::value_type;
+  using val_t        = val_vector_t::value_type;
+
+  auto constexpr num_items = 1000;
+  auto keys                = cuda::make_counting_iterator(key_t{0});
+  auto values              = cuda::make_counting_iterator(val_t{42});
+
+  thrust::device_vector<cuda::std::uint32_t> error_counter(1, 0);
+  auto const error_counter_ptr = thrust::raw_pointer_cast(error_counter.data());
+
+  key_vector_t unique_out(num_items);
+  val_vector_t aggregates_out(num_items);
+  auto [unique_out_end, aggregates_out_end] = thrust::reduce_by_key(
+    keys,
+    keys + num_items,
+    values,
+    unique_out.begin(),
+    aggregates_out.begin(),
+    check_valid_item_op{error_counter_ptr, num_items - 1});
+
+  // Verify that the number of unique keys is correct
+  const auto num_unique_out     = cuda::std::distance(unique_out.begin(), unique_out_end);
+  const auto num_aggregates_out = cuda::std::distance(aggregates_out.begin(), aggregates_out_end);
+  ASSERT_EQUAL(num_unique_out, num_items);
+  ASSERT_EQUAL(num_aggregates_out, num_items);
+
+  // Verify that the equality operator was never invoked on out-of-bounds items
+  ASSERT_EQUAL(error_counter[0], cuda::std::uint32_t{0});
+
+  // Verify that unique keys are correct
+  bool all_keys_correct = thrust::equal(unique_out.cbegin(), unique_out.cend(), keys);
+  ASSERT_EQUAL(all_keys_correct, true);
+
+  // Verify that the aggregates are correct
+  bool all_values_correct = thrust::equal(aggregates_out.cbegin(), aggregates_out.cend(), values);
+  ASSERT_EQUAL(all_values_correct, true);
+}
+
+DECLARE_UNITTEST(TestReduceByKeyWithCustomEqualityOp);
+
+void TestReduceByKeyWithDifferentAccumulatorT()
+{
+  using key_t          = cuda::std::uint32_t;
+  using val_t          = cuda::std::uint8_t;
+  using reduction_op_t = check_accumulator_t_op<cuda::std::uint32_t>;
+
+  auto constexpr num_items            = 20000;
+  auto constexpr expected_num_uniques = 1;
+  constexpr auto unique_key           = key_t{42U};
+  auto keys                           = cuda::make_constant_iterator(unique_key);
+  auto values                         = cuda::make_counting_iterator(val_t{0});
+
+  thrust::device_vector<cuda::std::uint32_t> error_counter(1, 0);
+  auto const error_counter_ptr = thrust::raw_pointer_cast(error_counter.data());
+
+  thrust::device_vector<key_t> unique_out(expected_num_uniques);
+  thrust::device_vector<val_t> aggregates_out(expected_num_uniques);
+  auto [unique_out_end, aggregates_out_end] = thrust::reduce_by_key(
+    keys,
+    keys + num_items,
+    values,
+    unique_out.begin(),
+    aggregates_out.begin(),
+    cuda::std::equal_to<>{},
+    reduction_op_t{error_counter_ptr});
+
+  // Verify that the number of unique keys is correct
+  auto num_unique_out     = cuda::std::distance(unique_out.begin(), unique_out_end);
+  auto num_aggregates_out = cuda::std::distance(aggregates_out.begin(), aggregates_out_end);
+  ASSERT_EQUAL(num_unique_out, expected_num_uniques);
+  ASSERT_EQUAL(num_aggregates_out, expected_num_uniques);
+
+  // Verify that the equality operator was never invoked on out-of-bounds items
+  ASSERT_EQUAL(error_counter[0], cuda::std::uint32_t{0});
+
+  // Verify that the unique key is correct
+  ASSERT_EQUAL(unique_out[0], unique_key);
+
+  // // Verify that the aggregate is correct
+  constexpr auto mod_val            = 0x01 << cuda::std::numeric_limits<val_t>::digits;
+  constexpr auto sum                = ((num_items * (num_items - 1)) / 2);
+  constexpr auto expected_aggregate = static_cast<val_t>(sum % mod_val);
+  ASSERT_EQUAL(aggregates_out[0], expected_aggregate);
+}
+
+DECLARE_UNITTEST(TestReduceByKeyWithDifferentAccumulatorT);

cccl_upstream/thrust/testing/cuda/reduce_into.cu

@@ -0,0 +1,124 @@
+#include <thrust/execution_policy.h>
+#include <thrust/reduce.h>
+
+#include <cuda/iterator>
+
+#include <unittest/unittest.h>
+
+template <typename ExecutionPolicy, typename InputIter, typename OutputIter, typename T>
+__global__ void reduce_into_kernel(ExecutionPolicy exec, InputIter first, InputIter last, OutputIter result, T init)
+{
+  thrust::reduce_into(exec, first, last, result, init);
+}
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename T, typename ExecutionPolicy>
+void TestReduceIntoDevice(ExecutionPolicy exec, const size_t n)
+{
+  thrust::host_vector<T> h_data   = unittest::random_integers<T>(n);
+  thrust::device_vector<T> d_data = h_data;
+
+  thrust::host_vector<T> h_result(1);
+  thrust::device_vector<T> d_result(1);
+
+  T init = 13;
+
+  thrust::reduce_into(h_data.begin(), h_data.end(), h_result.begin(), init);
+
+  reduce_into_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), d_result.begin(), init);
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(h_result, d_result);
+}
+
+template <typename T>
+struct TestReduceIntoDeviceSeq
+{
+  void operator()(const size_t n)
+  {
+    TestReduceIntoDevice<T>(thrust::seq, n);
+  }
+};
+VariableUnitTest<TestReduceIntoDeviceSeq, IntegralTypes> TestReduceIntoDeviceSeqInstance;
+
+template <typename T>
+struct TestReduceIntoDeviceDevice
+{
+  void operator()(const size_t n)
+  {
+    TestReduceIntoDevice<T>(thrust::device, n);
+  }
+};
+VariableUnitTest<TestReduceIntoDeviceDevice, IntegralTypes> TestReduceIntoDeviceDeviceInstance;
+
+template <typename T>
+struct TestReduceIntoDeviceNoSync
+{
+  void operator()(const size_t n)
+  {
+    TestReduceIntoDevice<T>(thrust::cuda::par_nosync, n);
+  }
+};
+VariableUnitTest<TestReduceIntoDeviceNoSync, IntegralTypes> TestReduceIntoDeviceNoSyncInstance;
+#endif
+
+template <typename ExecutionPolicy>
+void TestReduceIntoCudaStreams(ExecutionPolicy policy)
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector v = {1, -2, 3};
+
+  Vector o(1);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  auto streampolicy = policy.on(s);
+
+  // no initializer
+  thrust::reduce_into(streampolicy, v.begin(), v.end(), o.begin());
+
+  cudaStreamSynchronize(s);
+  ASSERT_EQUAL(o[0], 2);
+
+  // with initializer
+  thrust::reduce_into(streampolicy, v.begin(), v.end(), o.begin(), 10);
+
+  cudaStreamSynchronize(s);
+  ASSERT_EQUAL(o[0], 12);
+
+  cudaStreamDestroy(s);
+}
+
+void TestReduceIntoCudaStreamsSync()
+{
+  TestReduceIntoCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestReduceIntoCudaStreamsSync);
+
+void TestReduceIntoCudaStreamsNoSync()
+{
+  TestReduceIntoCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestReduceIntoCudaStreamsNoSync);
+
+#if defined(THRUST_RDC_ENABLED)
+void TestReduceIntoLargeInput()
+{
+  using T                 = unsigned long long;
+  using OffsetT           = std::size_t;
+  const OffsetT num_items = 1ull << 32;
+
+  cuda::constant_iterator<T> d_data(T{1});
+  thrust::device_vector<T> d_result(1);
+
+  reduce_into_kernel<<<1, 1>>>(thrust::device, d_data, d_data + num_items, d_result.begin(), T{});
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(num_items, d_result[0]);
+}
+DECLARE_UNITTEST(TestReduceIntoLargeInput);
+#endif

cccl_upstream/thrust/testing/cuda/remove.cu

@@ -0,0 +1,470 @@
+#include <thrust/execution_policy.h>
+#include <thrust/remove.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename T, typename Iterator2>
+__global__ void remove_kernel(ExecutionPolicy exec, Iterator first, Iterator last, T val, Iterator2 result)
+{
+  *result = thrust::remove(exec, first, last, val);
+}
+
+template <typename ExecutionPolicy, typename Iterator, typename Predicate, typename Iterator2>
+__global__ void remove_if_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Predicate pred, Iterator2 result)
+{
+  *result = thrust::remove_if(exec, first, last, pred);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Predicate, typename Iterator3>
+__global__ void remove_if_kernel(
+  ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 stencil_first, Predicate pred, Iterator3 result)
+{
+  *result = thrust::remove_if(exec, first, last, stencil_first, pred);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename T, typename Iterator3>
+__global__ void
+remove_copy_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result1, T val, Iterator3 result2)
+{
+  *result2 = thrust::remove_copy(exec, first, last, result1, val);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Predicate, typename Iterator3>
+__global__ void remove_copy_if_kernel(
+  ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result, Predicate pred, Iterator3 result_end)
+{
+  *result_end = thrust::remove_copy_if(exec, first, last, result, pred);
+}
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Predicate,
+          typename Iterator4>
+__global__ void remove_copy_if_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first,
+  Iterator1 last,
+  Iterator2 stencil_first,
+  Iterator3 result,
+  Predicate pred,
+  Iterator4 result_end)
+{
+  *result_end = thrust::remove_copy_if(exec, first, last, stencil_first, result, pred);
+}
+#endif
+
+template <typename T>
+struct is_even
+{
+  _CCCL_HOST_DEVICE bool operator()(T x)
+  {
+    return (static_cast<unsigned int>(x) & 1) == 0;
+  }
+};
+
+template <typename T>
+struct is_true
+{
+  _CCCL_HOST_DEVICE bool operator()(T x)
+  {
+    return x ? true : false;
+  }
+};
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy>
+void TestRemoveDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  using iterator = typename thrust::device_vector<int>::iterator;
+  thrust::device_vector<iterator> d_result(1);
+
+  size_t h_size = thrust::remove(h_data.begin(), h_data.end(), 0) - h_data.begin();
+
+  remove_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), 0, d_result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  size_t d_size = (iterator) d_result[0] - d_data.begin();
+
+  ASSERT_EQUAL(h_size, d_size);
+
+  h_data.resize(h_size);
+  d_data.resize(d_size);
+
+  ASSERT_EQUAL(h_data, d_data);
+}
+
+void TestRemoveDeviceSeq()
+{
+  TestRemoveDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestRemoveDeviceSeq);
+
+void TestRemoveDeviceDevice()
+{
+  TestRemoveDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestRemoveDeviceDevice);
+
+template <typename ExecutionPolicy>
+void TestRemoveIfDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  using iterator = typename thrust::device_vector<int>::iterator;
+  thrust::device_vector<iterator> d_result(1);
+
+  size_t h_size = thrust::remove_if(h_data.begin(), h_data.end(), is_true<int>()) - h_data.begin();
+
+  remove_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), is_true<int>(), d_result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  size_t d_size = (iterator) d_result[0] - d_data.begin();
+
+  ASSERT_EQUAL(h_size, d_size);
+
+  h_data.resize(h_size);
+  d_data.resize(d_size);
+
+  ASSERT_EQUAL(h_data, d_data);
+}
+
+void TestRemoveIfDeviceSeq()
+{
+  TestRemoveIfDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestRemoveIfDeviceSeq);
+
+void TestRemoveIfDeviceDevice()
+{
+  TestRemoveIfDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestRemoveIfDeviceDevice);
+
+template <typename ExecutionPolicy>
+void TestRemoveIfStencilDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  using iterator = typename thrust::device_vector<int>::iterator;
+  thrust::device_vector<iterator> d_result(1);
+
+  thrust::host_vector<bool> h_stencil   = unittest::random_integers<bool>(n);
+  thrust::device_vector<bool> d_stencil = h_stencil;
+
+  size_t h_size = thrust::remove_if(h_data.begin(), h_data.end(), h_stencil.begin(), is_true<int>()) - h_data.begin();
+
+  remove_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), d_stencil.begin(), is_true<int>(), d_result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  size_t d_size = (iterator) d_result[0] - d_data.begin();
+
+  ASSERT_EQUAL(h_size, d_size);
+
+  h_data.resize(h_size);
+  d_data.resize(d_size);
+
+  ASSERT_EQUAL(h_data, d_data);
+}
+
+void TestRemoveIfStencilDeviceSeq()
+{
+  TestRemoveIfStencilDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestRemoveIfStencilDeviceSeq);
+
+void TestRemoveIfStencilDeviceDevice()
+{
+  TestRemoveIfStencilDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestRemoveIfStencilDeviceDevice);
+
+template <typename ExecutionPolicy>
+void TestRemoveCopyDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  thrust::host_vector<int> h_result(n);
+  thrust::device_vector<int> d_result(n);
+
+  using iterator = typename thrust::device_vector<int>::iterator;
+  thrust::device_vector<iterator> d_new_end(1);
+
+  size_t h_size = thrust::remove_copy(h_data.begin(), h_data.end(), h_result.begin(), 0) - h_result.begin();
+
+  remove_copy_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), d_result.begin(), 0, d_new_end.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  size_t d_size = (iterator) d_new_end[0] - d_result.begin();
+
+  ASSERT_EQUAL(h_size, d_size);
+
+  h_result.resize(h_size);
+  d_result.resize(d_size);
+
+  ASSERT_EQUAL(h_result, d_result);
+}
+
+void TestRemoveCopyDeviceSeq()
+{
+  TestRemoveCopyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestRemoveCopyDeviceSeq);
+
+void TestRemoveCopyDeviceDevice()
+{
+  TestRemoveCopyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestRemoveCopyDeviceDevice);
+
+template <typename ExecutionPolicy>
+void TestRemoveCopyIfDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  thrust::host_vector<int> h_result(n);
+  thrust::device_vector<int> d_result(n);
+
+  using iterator = typename thrust::device_vector<int>::iterator;
+  thrust::device_vector<iterator> d_new_end(1);
+
+  size_t h_size =
+    thrust::remove_copy_if(h_data.begin(), h_data.end(), h_result.begin(), is_true<int>()) - h_result.begin();
+
+  remove_copy_if_kernel<<<1, 1>>>(
+    exec, d_data.begin(), d_data.end(), d_result.begin(), is_true<int>(), d_new_end.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  size_t d_size = (iterator) d_new_end[0] - d_result.begin();
+
+  ASSERT_EQUAL(h_size, d_size);
+
+  h_result.resize(h_size);
+  d_result.resize(d_size);
+
+  ASSERT_EQUAL(h_result, d_result);
+}
+
+void TestRemoveCopyIfDeviceSeq()
+{
+  TestRemoveCopyIfDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestRemoveCopyIfDeviceSeq);
+
+void TestRemoveCopyIfDeviceDevice()
+{
+  TestRemoveCopyIfDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestRemoveCopyIfDeviceDevice);
+
+template <typename ExecutionPolicy>
+void TestRemoveCopyIfStencilDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  thrust::host_vector<int> h_result(n);
+  thrust::device_vector<int> d_result(n);
+
+  using iterator = typename thrust::device_vector<int>::iterator;
+  thrust::device_vector<iterator> d_new_end(1);
+
+  thrust::host_vector<bool> h_stencil   = unittest::random_integers<bool>(n);
+  thrust::device_vector<bool> d_stencil = h_stencil;
+
+  size_t h_size =
+    thrust::remove_copy_if(h_data.begin(), h_data.end(), h_stencil.begin(), h_result.begin(), is_true<int>())
+    - h_result.begin();
+
+  remove_copy_if_kernel<<<1, 1>>>(
+    exec, d_data.begin(), d_data.end(), d_stencil.begin(), d_result.begin(), is_true<int>(), d_new_end.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  size_t d_size = (iterator) d_new_end[0] - d_result.begin();
+
+  ASSERT_EQUAL(h_size, d_size);
+
+  h_result.resize(h_size);
+  d_result.resize(d_size);
+
+  ASSERT_EQUAL(h_result, d_result);
+}
+
+void TestRemoveCopyIfStencilDeviceSeq()
+{
+  TestRemoveCopyIfStencilDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestRemoveCopyIfStencilDeviceSeq);
+
+void TestRemoveCopyIfStencilDeviceDevice()
+{
+  TestRemoveCopyIfStencilDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestRemoveCopyIfStencilDeviceDevice);
+#endif
+
+void TestRemoveCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{1, 2, 1, 3, 2};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  Vector::iterator end = thrust::remove(thrust::cuda::par.on(s), data.begin(), data.end(), (T) 2);
+
+  ASSERT_EQUAL(end - data.begin(), 3);
+  data.erase(end, data.end());
+
+  Vector ref{1, 1, 3};
+  ASSERT_EQUAL(data, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestRemoveCudaStreams);
+
+void TestRemoveCopyCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{1, 2, 1, 3, 2};
+
+  Vector result(5);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  Vector::iterator end = thrust::remove_copy(thrust::cuda::par.on(s), data.begin(), data.end(), result.begin(), (T) 2);
+
+  ASSERT_EQUAL(end - result.begin(), 3);
+  result.erase(end, result.end());
+
+  Vector ref{1, 1, 3};
+  ASSERT_EQUAL(result, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestRemoveCopyCudaStreams);
+
+void TestRemoveIfCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{1, 2, 1, 3, 2};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  Vector::iterator end = thrust::remove_if(thrust::cuda::par.on(s), data.begin(), data.end(), is_even<T>());
+
+  ASSERT_EQUAL(end - data.begin(), 3);
+  data.erase(end, data.end());
+
+  Vector ref{1, 1, 3};
+  ASSERT_EQUAL(data, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestRemoveIfCudaStreams);
+
+void TestRemoveIfStencilCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{1, 2, 1, 3, 2};
+
+  Vector stencil{0, 1, 0, 0, 1};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  Vector::iterator end =
+    thrust::remove_if(thrust::cuda::par.on(s), data.begin(), data.end(), stencil.begin(), ::cuda::std::identity{});
+
+  ASSERT_EQUAL(end - data.begin(), 3);
+  data.erase(end, data.end());
+
+  Vector ref{1, 1, 3};
+  ASSERT_EQUAL(data, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestRemoveIfStencilCudaStreams);
+
+void TestRemoveCopyIfCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{1, 2, 1, 3, 2};
+
+  Vector result(5);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  Vector::iterator end =
+    thrust::remove_copy_if(thrust::cuda::par.on(s), data.begin(), data.end(), result.begin(), is_even<T>());
+
+  ASSERT_EQUAL(end - result.begin(), 3);
+  result.erase(end, result.end());
+
+  Vector ref{1, 1, 3};
+  ASSERT_EQUAL(result, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestRemoveCopyIfCudaStreams);
+
+void TestRemoveCopyIfStencilCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{1, 2, 1, 3, 2};
+
+  Vector stencil{0, 1, 0, 0, 1};
+
+  Vector result(5);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  Vector::iterator end = thrust::remove_copy_if(
+    thrust::cuda::par.on(s), data.begin(), data.end(), stencil.begin(), result.begin(), ::cuda::std::identity{});
+
+  ASSERT_EQUAL(end - result.begin(), 3);
+  result.erase(end, result.end());
+
+  Vector ref{1, 1, 3};
+  ASSERT_EQUAL(result, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestRemoveCopyIfStencilCudaStreams);

cccl_upstream/thrust/testing/cuda/replace.cu

@@ -0,0 +1,283 @@
+#include <thrust/execution_policy.h>
+#include <thrust/replace.h>
+
+#include <unittest/unittest.h>
+
+template <typename T>
+struct less_than_five
+{
+  _CCCL_HOST_DEVICE bool operator()(const T& val) const
+  {
+    return val < 5;
+  }
+};
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename T1, typename T2>
+__global__ void replace_kernel(ExecutionPolicy exec, Iterator first, Iterator last, T1 old_value, T2 new_value)
+{
+  thrust::replace(exec, first, last, old_value, new_value);
+}
+
+template <typename T, typename ExecutionPolicy>
+void TestReplaceDevice(ExecutionPolicy exec, const size_t n)
+{
+  thrust::host_vector<T> h_data   = unittest::random_samples<T>(n);
+  thrust::device_vector<T> d_data = h_data;
+
+  T old_value = 0;
+  T new_value = 1;
+
+  thrust::replace(h_data.begin(), h_data.end(), old_value, new_value);
+
+  replace_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), old_value, new_value);
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_ALMOST_EQUAL(h_data, d_data);
+}
+
+template <typename T>
+void TestReplaceDeviceSeq(const size_t n)
+{
+  TestReplaceDevice<T>(thrust::seq, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestReplaceDeviceSeq);
+
+template <typename T>
+void TestReplaceDeviceDevice(const size_t n)
+{
+  TestReplaceDevice<T>(thrust::device, n);
+}
+DECLARE_VARIABLE_UNITTEST(TestReplaceDeviceDevice);
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename T1, typename T2>
+__global__ void
+replace_copy_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result, T1 old_value, T2 new_value)
+{
+  thrust::replace_copy(exec, first, last, result, old_value, new_value);
+}
+
+template <typename ExecutionPolicy>
+void TestReplaceCopyDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  int old_value = 0;
+  int new_value = 1;
+
+  thrust::host_vector<int> h_dest(n);
+  thrust::device_vector<int> d_dest(n);
+
+  thrust::replace_copy(h_data.begin(), h_data.end(), h_dest.begin(), old_value, new_value);
+
+  replace_copy_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), d_dest.begin(), old_value, new_value);
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_ALMOST_EQUAL(h_data, d_data);
+  ASSERT_ALMOST_EQUAL(h_dest, d_dest);
+}
+
+void TestReplaceCopyDeviceSeq()
+{
+  TestReplaceCopyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestReplaceCopyDeviceSeq);
+
+void TestReplaceCopyDeviceDevice()
+{
+  TestReplaceCopyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestReplaceCopyDeviceDevice);
+
+template <typename ExecutionPolicy, typename Iterator, typename Predicate, typename T>
+__global__ void replace_if_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Predicate pred, T new_value)
+{
+  thrust::replace_if(exec, first, last, pred, new_value);
+}
+
+template <typename ExecutionPolicy>
+void TestReplaceIfDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  thrust::replace_if(h_data.begin(), h_data.end(), less_than_five<int>(), 0);
+
+  replace_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), less_than_five<int>(), 0);
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_ALMOST_EQUAL(h_data, d_data);
+}
+
+void TestReplaceIfDeviceSeq()
+{
+  TestReplaceIfDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestReplaceIfDeviceSeq);
+
+void TestReplaceIfDeviceDevice()
+{
+  TestReplaceIfDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestReplaceIfDeviceDevice);
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Predicate, typename T>
+__global__ void replace_if_kernel(
+  ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 stencil_first, Predicate pred, T new_value)
+{
+  thrust::replace_if(exec, first, last, stencil_first, pred, new_value);
+}
+
+template <typename ExecutionPolicy>
+void TestReplaceIfStencilDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  thrust::host_vector<int> h_stencil   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_stencil = h_stencil;
+
+  thrust::replace_if(h_data.begin(), h_data.end(), h_stencil.begin(), less_than_five<int>(), 0);
+
+  replace_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), d_stencil.begin(), less_than_five<int>(), 0);
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_ALMOST_EQUAL(h_data, d_data);
+}
+
+void TestReplaceIfStencilDeviceSeq()
+{
+  TestReplaceIfStencilDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestReplaceIfStencilDeviceSeq);
+
+void TestReplaceIfStencilDeviceDevice()
+{
+  TestReplaceIfStencilDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestReplaceIfStencilDeviceDevice);
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Predicate, typename T>
+__global__ void replace_copy_if_kernel(
+  ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result, Predicate pred, T new_value)
+{
+  thrust::replace_copy_if(exec, first, last, result, pred, new_value);
+}
+
+template <typename ExecutionPolicy>
+void TestReplaceCopyIfDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  thrust::host_vector<int> h_dest(n);
+  thrust::device_vector<int> d_dest(n);
+
+  thrust::replace_copy_if(h_data.begin(), h_data.end(), h_dest.begin(), less_than_five<int>(), 0);
+
+  replace_copy_if_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), d_dest.begin(), less_than_five<int>(), 0);
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_ALMOST_EQUAL(h_data, d_data);
+  ASSERT_ALMOST_EQUAL(h_dest, d_dest);
+}
+
+void TestReplaceCopyIfDeviceSeq()
+{
+  TestReplaceCopyIfDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestReplaceCopyIfDeviceSeq);
+
+void TestReplaceCopyIfDeviceDevice()
+{
+  TestReplaceCopyIfDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestReplaceCopyIfDeviceDevice);
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Predicate,
+          typename T>
+__global__ void replace_copy_if_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first,
+  Iterator1 last,
+  Iterator2 stencil_first,
+  Iterator3 result,
+  Predicate pred,
+  T new_value)
+{
+  thrust::replace_copy_if(exec, first, last, stencil_first, result, pred, new_value);
+}
+
+template <typename ExecutionPolicy>
+void TestReplaceCopyIfStencilDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  thrust::host_vector<int> h_stencil   = unittest::random_samples<int>(n);
+  thrust::device_vector<int> d_stencil = h_stencil;
+
+  thrust::host_vector<int> h_dest(n);
+  thrust::device_vector<int> d_dest(n);
+
+  thrust::replace_copy_if(h_data.begin(), h_data.end(), h_stencil.begin(), h_dest.begin(), less_than_five<int>(), 0);
+
+  replace_copy_if_kernel<<<1, 1>>>(
+    exec, d_data.begin(), d_data.end(), d_stencil.begin(), d_dest.begin(), less_than_five<int>(), 0);
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_ALMOST_EQUAL(h_data, d_data);
+  ASSERT_ALMOST_EQUAL(h_dest, d_dest);
+}
+
+void TestReplaceCopyIfStencilDeviceSeq()
+{
+  TestReplaceCopyIfStencilDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestReplaceCopyIfStencilDeviceSeq);
+
+void TestReplaceCopyIfStencilDeviceDevice()
+{
+  TestReplaceCopyIfStencilDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestReplaceCopyIfStencilDeviceDevice);
+#endif
+
+void TestReplaceCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{1, 2, 1, 3, 2};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::replace(thrust::cuda::par.on(s), data.begin(), data.end(), (T) 1, (T) 4);
+  thrust::replace(thrust::cuda::par.on(s), data.begin(), data.end(), (T) 2, (T) 5);
+
+  cudaStreamSynchronize(s);
+
+  Vector result{4, 5, 4, 3, 5};
+
+  ASSERT_EQUAL(data, result);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestReplaceCudaStreams);

cccl_upstream/thrust/testing/cuda/reverse.cu

@@ -0,0 +1,119 @@
+#include <thrust/execution_policy.h>
+#include <thrust/reverse.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator>
+__global__ void reverse_kernel(ExecutionPolicy exec, Iterator first, Iterator last)
+{
+  thrust::reverse(exec, first, last);
+}
+
+template <typename ExecutionPolicy>
+void TestReverseDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_integers<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  thrust::reverse(h_data.begin(), h_data.end());
+
+  reverse_kernel<<<1, 1>>>(exec, raw_pointer_cast(d_data.data()), raw_pointer_cast(d_data.data() + d_data.size()));
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(h_data, d_data);
+};
+
+void TestReverseDeviceSeq()
+{
+  TestReverseDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestReverseDeviceSeq);
+
+void TestReverseDeviceDevice()
+{
+  TestReverseDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestReverseDeviceDevice);
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
+__global__ void reverse_copy_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result)
+{
+  thrust::reverse_copy(exec, first, last, result);
+}
+
+template <typename ExecutionPolicy>
+void TestReverseCopyDevice(ExecutionPolicy exec)
+{
+  size_t n                          = 1000;
+  thrust::host_vector<int> h_data   = unittest::random_integers<int>(n);
+  thrust::device_vector<int> d_data = h_data;
+
+  thrust::host_vector<int> h_result(n);
+  thrust::device_vector<int> d_result(n);
+
+  thrust::reverse_copy(h_data.begin(), h_data.end(), h_result.begin());
+
+  reverse_copy_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), d_result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(h_result, d_result);
+};
+
+void TestReverseCopyDeviceSeq()
+{
+  TestReverseCopyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestReverseCopyDeviceSeq);
+
+void TestReverseCopyDeviceDevice()
+{
+  TestReverseCopyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestReverseCopyDeviceDevice);
+#endif
+
+void TestReverseCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  Vector data{1, 2, 3, 4, 5};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::reverse(thrust::cuda::par.on(s), data.begin(), data.end());
+
+  cudaStreamSynchronize(s);
+
+  Vector ref{5, 4, 3, 2, 1};
+
+  ASSERT_EQUAL(ref, data);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestReverseCudaStreams);
+
+void TestReverseCopyCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  Vector data{1, 2, 3, 4, 5};
+
+  Vector result(5);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::reverse_copy(thrust::cuda::par.on(s), data.begin(), data.end(), result.begin());
+
+  cudaStreamSynchronize(s);
+
+  Vector ref{5, 4, 3, 2, 1};
+
+  ASSERT_EQUAL(ref, result);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestReverseCopyCudaStreams);

cccl_upstream/thrust/testing/cuda/scan.cu

@@ -0,0 +1,265 @@
+#include <thrust/functional.h>
+#include <thrust/scan.h>
+
+#include <cstdio>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
+__global__ void inclusive_scan_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result)
+{
+  thrust::inclusive_scan(exec, first, last, result);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename T, typename Pred>
+__global__ void
+inclusive_scan_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result, T init, Pred pred)
+{
+  thrust::inclusive_scan(exec, first, last, result, init, pred);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
+__global__ void exclusive_scan_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result)
+{
+  thrust::exclusive_scan(exec, first, last, result);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename T>
+__global__ void exclusive_scan_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result, T init)
+{
+  thrust::exclusive_scan(exec, first, last, result, init);
+}
+
+template <typename T, typename ExecutionPolicy>
+void TestScanDevice(ExecutionPolicy exec, const size_t n)
+{
+  thrust::host_vector<T> h_input   = unittest::random_integers<T>(n);
+  thrust::device_vector<T> d_input = h_input;
+
+  thrust::host_vector<T> h_output(n);
+  thrust::device_vector<T> d_output(n);
+
+  thrust::inclusive_scan(h_input.begin(), h_input.end(), h_output.begin());
+
+  inclusive_scan_kernel<<<1, 1>>>(exec, d_input.begin(), d_input.end(), d_output.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(d_output, h_output);
+
+  thrust::inclusive_scan(h_input.begin(), h_input.end(), h_output.begin(), (T) 11, ::cuda::std::plus<T>{});
+
+  inclusive_scan_kernel<<<1, 1>>>(
+    exec, d_input.begin(), d_input.end(), d_output.begin(), (T) 11, ::cuda::std::plus<T>{});
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(d_output, h_output);
+
+  thrust::exclusive_scan(h_input.begin(), h_input.end(), h_output.begin());
+
+  exclusive_scan_kernel<<<1, 1>>>(exec, d_input.begin(), d_input.end(), d_output.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(d_output, h_output);
+
+  thrust::exclusive_scan(h_input.begin(), h_input.end(), h_output.begin(), (T) 11);
+
+  exclusive_scan_kernel<<<1, 1>>>(exec, d_input.begin(), d_input.end(), d_output.begin(), (T) 11);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(d_output, h_output);
+
+  // in-place scans
+  h_output = h_input;
+  d_output = d_input;
+
+  thrust::inclusive_scan(h_output.begin(), h_output.end(), h_output.begin());
+
+  inclusive_scan_kernel<<<1, 1>>>(exec, d_output.begin(), d_output.end(), d_output.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(d_output, h_output);
+
+  h_output = h_input;
+  d_output = d_input;
+
+  thrust::exclusive_scan(h_output.begin(), h_output.end(), h_output.begin());
+
+  exclusive_scan_kernel<<<1, 1>>>(exec, d_output.begin(), d_output.end(), d_output.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(d_output, h_output);
+}
+
+template <typename T>
+struct TestScanDeviceSeq
+{
+  void operator()(const size_t n)
+  {
+    TestScanDevice<T>(thrust::seq, n);
+  }
+};
+VariableUnitTest<TestScanDeviceSeq, IntegralTypes> TestScanDeviceSeqInstance;
+
+template <typename T>
+struct TestScanDeviceDevice
+{
+  void operator()(const size_t n)
+  {
+    TestScanDevice<T>(thrust::device, n);
+  }
+};
+VariableUnitTest<TestScanDeviceDevice, IntegralTypes> TestScanDeviceDeviceInstance;
+#endif
+
+void TestScanCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector::iterator iter;
+
+  Vector input{1, 3, -2, 4, -5};
+  Vector result{1, 4, 2, 6, 1};
+  Vector output(5);
+
+  Vector input_copy(input);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+  // inclusive scan
+
+  iter = thrust::inclusive_scan(thrust::cuda::par.on(s), input.begin(), input.end(), output.begin());
+
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(output, result);
+
+  // exclusive scan
+  iter = thrust::exclusive_scan(thrust::cuda::par.on(s), input.begin(), input.end(), output.begin(), 0);
+  cudaStreamSynchronize(s);
+
+  result = {0, 1, 4, 2, 6};
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(output, result);
+
+  // exclusive scan with init
+  iter = thrust::exclusive_scan(thrust::cuda::par.on(s), input.begin(), input.end(), output.begin(), 3);
+  cudaStreamSynchronize(s);
+
+  result = {3, 4, 7, 5, 9};
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(output, result);
+
+  // inclusive scan with op
+  iter =
+    thrust::inclusive_scan(thrust::cuda::par.on(s), input.begin(), input.end(), output.begin(), ::cuda::std::plus<T>());
+  cudaStreamSynchronize(s);
+
+  result = {1, 4, 2, 6, 1};
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(output, result);
+
+  // inclusive scan with init and op
+  iter = thrust::inclusive_scan(
+    thrust::cuda::par.on(s), input.begin(), input.end(), output.begin(), 3, ::cuda::std::plus<T>());
+  cudaStreamSynchronize(s);
+
+  result = {4, 7, 5, 9, 4};
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(output, result);
+
+  // exclusive scan with init and op
+  iter = thrust::exclusive_scan(
+    thrust::cuda::par.on(s), input.begin(), input.end(), output.begin(), 3, ::cuda::std::plus<T>());
+  cudaStreamSynchronize(s);
+
+  result = {3, 4, 7, 5, 9};
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(output, result);
+
+  // inplace inclusive scan
+  input = input_copy;
+  iter  = thrust::inclusive_scan(thrust::cuda::par.on(s), input.begin(), input.end(), input.begin());
+  cudaStreamSynchronize(s);
+
+  result = {1, 4, 2, 6, 1};
+  ASSERT_EQUAL(std::size_t(iter - input.begin()), input.size());
+  ASSERT_EQUAL(input, result);
+
+  // inplace exclusive scan with init
+  input = input_copy;
+  iter  = thrust::exclusive_scan(thrust::cuda::par.on(s), input.begin(), input.end(), input.begin(), 3);
+  cudaStreamSynchronize(s);
+
+  result = {3, 4, 7, 5, 9};
+  ASSERT_EQUAL(std::size_t(iter - input.begin()), input.size());
+  ASSERT_EQUAL(input, result);
+
+  // inplace exclusive scan with implicit init=0
+  input = input_copy;
+  iter  = thrust::exclusive_scan(thrust::cuda::par.on(s), input.begin(), input.end(), input.begin());
+  cudaStreamSynchronize(s);
+
+  result = {0, 1, 4, 2, 6};
+  ASSERT_EQUAL(std::size_t(iter - input.begin()), input.size());
+  ASSERT_EQUAL(input, result);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestScanCudaStreams);
+
+template <typename T>
+struct const_ref_plus_mod3
+{
+  T* table;
+
+  const_ref_plus_mod3(T* table)
+      : table(table)
+  {}
+
+  _CCCL_HOST_DEVICE const T& operator()(T a, T b)
+  {
+    return table[(int) (a + b)];
+  }
+};
+
+static void TestInclusiveScanWithConstAccumulator()
+{
+  // add numbers modulo 3 with external lookup table
+  thrust::device_vector<int> data{0, 1, 2, 1, 2, 0, 1};
+
+  thrust::device_vector<int> table{0, 1, 2, 0, 1, 2};
+
+  thrust::inclusive_scan(
+    data.begin(), data.end(), data.begin(), const_ref_plus_mod3<int>(thrust::raw_pointer_cast(&table[0])));
+
+  thrust::device_vector<int> ref{0, 1, 0, 1, 0, 0, 1};
+  ASSERT_EQUAL(data, ref);
+}
+DECLARE_UNITTEST(TestInclusiveScanWithConstAccumulator);

cccl_upstream/thrust/testing/cuda/scan_by_key.cu

@@ -0,0 +1,229 @@
+#include <thrust/execution_policy.h>
+#include <thrust/functional.h>
+#include <thrust/scan.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Iterator3>
+__global__ void inclusive_scan_by_key_kernel(
+  ExecutionPolicy exec, Iterator1 keys_first, Iterator1 keys_last, Iterator2 values_first, Iterator3 result)
+{
+  thrust::inclusive_scan_by_key(exec, keys_first, keys_last, values_first, result);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Iterator3>
+__global__ void exclusive_scan_by_key_kernel(
+  ExecutionPolicy exec, Iterator1 keys_first, Iterator1 keys_last, Iterator2 values_first, Iterator3 result)
+{
+  thrust::exclusive_scan_by_key(exec, keys_first, keys_last, values_first, result);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Iterator3, typename T>
+__global__ void exclusive_scan_by_key_kernel(
+  ExecutionPolicy exec, Iterator1 keys_first, Iterator1 keys_last, Iterator2 values_first, Iterator3 result, T init)
+{
+  thrust::exclusive_scan_by_key(exec, keys_first, keys_last, values_first, result, init);
+}
+
+template <typename ExecutionPolicy>
+void TestScanByKeyDevice(ExecutionPolicy exec)
+{
+  size_t n = 1000;
+
+  thrust::host_vector<int> h_keys(n);
+  for (size_t i = 0, k = 0; i < n; i++)
+  {
+    h_keys[i] = static_cast<int>(k);
+    if (rand() % 10 == 0)
+    {
+      k++;
+    }
+  }
+  thrust::device_vector<int> d_keys = h_keys;
+
+  thrust::host_vector<int> h_vals = unittest::random_integers<int>(n);
+  for (size_t i = 0; i < n; i++)
+  {
+    h_vals[i] = i % 10;
+  }
+  thrust::device_vector<int> d_vals = h_vals;
+
+  thrust::host_vector<int> h_output(n);
+  thrust::device_vector<int> d_output(n);
+
+  thrust::inclusive_scan_by_key(h_keys.begin(), h_keys.end(), h_vals.begin(), h_output.begin());
+  inclusive_scan_by_key_kernel<<<1, 1>>>(exec, d_keys.begin(), d_keys.end(), d_vals.begin(), d_output.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+  ASSERT_EQUAL(d_output, h_output);
+
+  thrust::exclusive_scan_by_key(h_keys.begin(), h_keys.end(), h_vals.begin(), h_output.begin());
+  exclusive_scan_by_key_kernel<<<1, 1>>>(exec, d_keys.begin(), d_keys.end(), d_vals.begin(), d_output.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+  ASSERT_EQUAL(d_output, h_output);
+
+  thrust::exclusive_scan_by_key(h_keys.begin(), h_keys.end(), h_vals.begin(), h_output.begin(), 11);
+  exclusive_scan_by_key_kernel<<<1, 1>>>(exec, d_keys.begin(), d_keys.end(), d_vals.begin(), d_output.begin(), 11);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+  ASSERT_EQUAL(d_output, h_output);
+
+  // in-place scans: in/out values aliasing
+  h_output = h_vals;
+  d_output = d_vals;
+  thrust::inclusive_scan_by_key(h_keys.begin(), h_keys.end(), h_output.begin(), h_output.begin());
+  inclusive_scan_by_key_kernel<<<1, 1>>>(exec, d_keys.begin(), d_keys.end(), d_output.begin(), d_output.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+  ASSERT_EQUAL(d_output, h_output);
+
+  h_output = h_vals;
+  d_output = d_vals;
+  thrust::exclusive_scan_by_key(h_keys.begin(), h_keys.end(), h_output.begin(), h_output.begin(), 11);
+  exclusive_scan_by_key_kernel<<<1, 1>>>(exec, d_keys.begin(), d_keys.end(), d_output.begin(), d_output.begin(), 11);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+  ASSERT_EQUAL(d_output, h_output);
+
+  // in-place scans: keys/values aliasing
+  thrust::inclusive_scan_by_key(h_keys.begin(), h_keys.end(), h_vals.begin(), h_output.begin());
+  inclusive_scan_by_key_kernel<<<1, 1>>>(exec, d_keys.begin(), d_keys.end(), d_vals.begin(), d_keys.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+  ASSERT_EQUAL(d_keys, h_output);
+
+  d_keys = h_keys;
+  thrust::exclusive_scan_by_key(h_keys.begin(), h_keys.end(), h_vals.begin(), h_output.begin(), 11);
+  exclusive_scan_by_key_kernel<<<1, 1>>>(exec, d_keys.begin(), d_keys.end(), d_vals.begin(), d_keys.begin(), 11);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+  ASSERT_EQUAL(d_keys, h_output);
+}
+
+void TestScanByKeyDeviceSeq()
+{
+  TestScanByKeyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestScanByKeyDeviceSeq);
+
+void TestScanByKeyDeviceDevice()
+{
+  TestScanByKeyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestScanByKeyDeviceDevice);
+#endif
+
+void TestInclusiveScanByKeyCudaStreams()
+{
+  using Vector   = thrust::device_vector<int>;
+  using T        = Vector::value_type;
+  using Iterator = Vector::iterator;
+
+  Vector keys{0, 1, 1, 1, 2, 3, 3};
+  Vector vals{1, 2, 3, 4, 5, 6, 7};
+
+  Vector output(7, 0);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  Iterator iter =
+    thrust::inclusive_scan_by_key(thrust::cuda::par.on(s), keys.begin(), keys.end(), vals.begin(), output.begin());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL_QUIET(iter, output.end());
+
+  Vector ref{1, 2, 5, 9, 5, 6, 13};
+  ASSERT_EQUAL(output, ref);
+
+  thrust::inclusive_scan_by_key(
+    thrust::cuda::par.on(s),
+    keys.begin(),
+    keys.end(),
+    vals.begin(),
+    output.begin(),
+    ::cuda::std::equal_to<T>(),
+    ::cuda::std::multiplies<T>());
+  cudaStreamSynchronize(s);
+
+  ref = {1, 2, 6, 24, 5, 6, 42};
+  ASSERT_EQUAL(output, ref);
+
+  thrust::inclusive_scan_by_key(
+    thrust::cuda::par.on(s), keys.begin(), keys.end(), vals.begin(), output.begin(), ::cuda::std::equal_to<T>());
+  cudaStreamSynchronize(s);
+
+  ref = {1, 2, 5, 9, 5, 6, 13};
+  ASSERT_EQUAL(output, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestInclusiveScanByKeyCudaStreams);
+
+void TestExclusiveScanByKeyCudaStreams()
+{
+  using Vector   = thrust::device_vector<int>;
+  using T        = Vector::value_type;
+  using Iterator = Vector::iterator;
+
+  Vector keys{0, 1, 1, 1, 2, 3, 3};
+  Vector vals{1, 2, 3, 4, 5, 6, 7};
+
+  Vector output(7, 0);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  Iterator iter =
+    thrust::exclusive_scan_by_key(thrust::cuda::par.on(s), keys.begin(), keys.end(), vals.begin(), output.begin());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL_QUIET(iter, output.end());
+
+  Vector ref{0, 0, 2, 5, 0, 0, 6};
+  ASSERT_EQUAL(output, ref);
+
+  thrust::exclusive_scan_by_key(thrust::cuda::par.on(s), keys.begin(), keys.end(), vals.begin(), output.begin(), T(10));
+  cudaStreamSynchronize(s);
+
+  ref = {10, 10, 12, 15, 10, 10, 16};
+  ASSERT_EQUAL(output, ref);
+
+  thrust::exclusive_scan_by_key(
+    thrust::cuda::par.on(s),
+    keys.begin(),
+    keys.end(),
+    vals.begin(),
+    output.begin(),
+    T(10),
+    ::cuda::std::equal_to<T>(),
+    ::cuda::std::multiplies<T>());
+  cudaStreamSynchronize(s);
+
+  ref = {10, 10, 20, 60, 10, 10, 60};
+  ASSERT_EQUAL(output, ref);
+
+  thrust::exclusive_scan_by_key(
+    thrust::cuda::par.on(s), keys.begin(), keys.end(), vals.begin(), output.begin(), T(10), ::cuda::std::equal_to<T>());
+  cudaStreamSynchronize(s);
+
+  ref = {10, 10, 12, 15, 10, 10, 16};
+  ASSERT_EQUAL(output, ref);
+}
+DECLARE_UNITTEST(TestExclusiveScanByKeyCudaStreams);

cccl_upstream/thrust/testing/cuda/scatter.cu

@@ -0,0 +1,178 @@
+#include <thrust/execution_policy.h>
+#include <thrust/scatter.h>
+
+#include <algorithm>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Iterator3>
+__global__ void
+scatter_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 map_first, Iterator3 result)
+{
+  thrust::scatter(exec, first, last, map_first, result);
+}
+
+template <typename ExecutionPolicy>
+void TestScatterDevice(ExecutionPolicy exec)
+{
+  size_t n                 = 1000;
+  const size_t output_size = std::min((size_t) 10, 2 * n);
+
+  thrust::host_vector<int> h_input(n, 1);
+  thrust::device_vector<int> d_input(n, 1);
+
+  thrust::host_vector<unsigned int> h_map = unittest::random_integers<unsigned int>(n);
+
+  for (size_t i = 0; i < n; i++)
+  {
+    h_map[i] = h_map[i] % output_size;
+  }
+
+  thrust::device_vector<unsigned int> d_map = h_map;
+
+  thrust::host_vector<int> h_output(output_size, 0);
+  thrust::device_vector<int> d_output(output_size, 0);
+
+  thrust::scatter(h_input.begin(), h_input.end(), h_map.begin(), h_output.begin());
+
+  scatter_kernel<<<1, 1>>>(exec, d_input.begin(), d_input.end(), d_map.begin(), d_output.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(h_output, d_output);
+}
+
+void TestScatterDeviceSeq()
+{
+  TestScatterDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestScatterDeviceSeq);
+
+void TestScatterDeviceDevice()
+{
+  TestScatterDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestScatterDeviceDevice);
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Iterator4,
+          typename Function>
+__global__ void scatter_if_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first,
+  Iterator1 last,
+  Iterator2 map_first,
+  Iterator3 stencil_first,
+  Iterator4 result,
+  Function f)
+{
+  thrust::scatter_if(exec, first, last, map_first, stencil_first, result, f);
+}
+
+template <typename T>
+struct is_even_scatter_if
+{
+  _CCCL_HOST_DEVICE bool operator()(const T i) const
+  {
+    return (i % 2) == 0;
+  }
+};
+
+template <typename ExecutionPolicy>
+void TestScatterIfDevice(ExecutionPolicy exec)
+{
+  size_t n                 = 1000;
+  const size_t output_size = std::min((size_t) 10, 2 * n);
+
+  thrust::host_vector<int> h_input(n, 1);
+  thrust::device_vector<int> d_input(n, 1);
+
+  thrust::host_vector<unsigned int> h_map = unittest::random_integers<unsigned int>(n);
+
+  for (size_t i = 0; i < n; i++)
+  {
+    h_map[i] = h_map[i] % output_size;
+  }
+
+  thrust::device_vector<unsigned int> d_map = h_map;
+
+  thrust::host_vector<int> h_output(output_size, 0);
+  thrust::device_vector<int> d_output(output_size, 0);
+
+  thrust::scatter_if(
+    h_input.begin(), h_input.end(), h_map.begin(), h_map.begin(), h_output.begin(), is_even_scatter_if<unsigned int>());
+
+  scatter_if_kernel<<<1, 1>>>(
+    exec,
+    d_input.begin(),
+    d_input.end(),
+    d_map.begin(),
+    d_map.begin(),
+    d_output.begin(),
+    is_even_scatter_if<unsigned int>());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(h_output, d_output);
+}
+
+void TestScatterIfDeviceSeq()
+{
+  TestScatterIfDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestScatterIfDeviceSeq);
+
+void TestScatterIfDeviceDevice()
+{
+  TestScatterIfDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestScatterIfDeviceDevice);
+#endif
+
+void TestScatterCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector map{6, 3, 1, 7, 2}; // scatter indices
+  Vector src{0, 1, 2, 3, 4}; // source vector
+  Vector dst(8, 0); // destination vector
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::scatter(thrust::cuda::par.on(s), src.begin(), src.end(), map.begin(), dst.begin());
+
+  cudaStreamSynchronize(s);
+
+  Vector ref{0, 2, 4, 1, 0, 0, 0, 3};
+  ASSERT_EQUAL(dst, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestScatterCudaStreams);
+
+void TestScatterIfCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector flg{0, 1, 0, 1, 0}; // predicate array
+  Vector map{6, 3, 1, 7, 2}; // scatter indices
+  Vector src{0, 1, 2, 3, 4}; // source vector
+  Vector dst(8); // destination vector
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::scatter_if(thrust::cuda::par.on(s), src.begin(), src.end(), map.begin(), flg.begin(), dst.begin());
+  cudaStreamSynchronize(s);
+
+  Vector ref{0, 0, 0, 1, 0, 0, 0, 3};
+  ASSERT_EQUAL(dst, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestScatterIfCudaStreams);

cccl_upstream/thrust/testing/cuda/sequence.cu

@@ -0,0 +1,100 @@
+#include <thrust/execution_policy.h>
+#include <thrust/sequence.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator>
+__global__ void sequence_kernel(ExecutionPolicy exec, Iterator first, Iterator last)
+{
+  thrust::sequence(exec, first, last);
+}
+
+template <typename ExecutionPolicy, typename Iterator, typename T>
+__global__ void sequence_kernel(ExecutionPolicy exec, Iterator first, Iterator last, T init)
+{
+  thrust::sequence(exec, first, last, init);
+}
+
+template <typename ExecutionPolicy, typename Iterator, typename T>
+__global__ void sequence_kernel(ExecutionPolicy exec, Iterator first, Iterator last, T init, T step)
+{
+  thrust::sequence(exec, first, last, init, step);
+}
+
+template <typename ExecutionPolicy>
+void TestSequenceDevice(ExecutionPolicy exec)
+{
+  thrust::device_vector<int> v(5);
+
+  sequence_kernel<<<1, 1>>>(exec, v.begin(), v.end());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  thrust::device_vector<int> ref{0, 1, 2, 3, 4};
+  ASSERT_EQUAL(v, ref);
+
+  sequence_kernel<<<1, 1>>>(exec, v.begin(), v.end(), 10);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ref = {10, 11, 12, 13, 14};
+  ASSERT_EQUAL(v, ref);
+
+  sequence_kernel<<<1, 1>>>(exec, v.begin(), v.end(), 10, 2);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ref = {10, 12, 14, 16, 18};
+  ASSERT_EQUAL(v, ref);
+}
+
+void TestSequenceDeviceSeq()
+{
+  TestSequenceDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestSequenceDeviceSeq);
+
+void TestSequenceDeviceDevice()
+{
+  TestSequenceDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestSequenceDeviceDevice);
+#endif
+
+void TestSequenceCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector v(5);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::sequence(thrust::cuda::par.on(s), v.begin(), v.end());
+  cudaStreamSynchronize(s);
+
+  Vector ref{0, 1, 2, 3, 4};
+  ASSERT_EQUAL(v, ref);
+
+  thrust::sequence(thrust::cuda::par.on(s), v.begin(), v.end(), 10);
+  cudaStreamSynchronize(s);
+
+  ref = {10, 11, 12, 13, 14};
+  ASSERT_EQUAL(v, ref);
+
+  thrust::sequence(thrust::cuda::par.on(s), v.begin(), v.end(), 10, 2);
+  cudaStreamSynchronize(s);
+
+  ref = {10, 12, 14, 16, 18};
+  ASSERT_EQUAL(v, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestSequenceCudaStreams);

cccl_upstream/thrust/testing/cuda/set_difference.cu

@@ -0,0 +1,67 @@
+#include <thrust/execution_policy.h>
+#include <thrust/set_operations.h>
+
+#include <cuda/buffer>
+#include <cuda/cccl_runtime_test_helper.cuh>
+#include <cuda/launch>
+#include <cuda/std/initializer_list>
+#include <cuda/stream>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+struct set_difference_kernel
+{
+  template <typename ExecutionPolicy, typename Input1, typename Input2, typename Output>
+  __device__ void operator()(ExecutionPolicy exec, Input1 a, Input2 b, Output result) const
+  {
+    const auto end = thrust::set_difference(exec, a.begin(), a.end(), b.begin(), b.end(), result.begin());
+    TEST_ASSERT_DEVICE(end == result.end());
+  }
+};
+
+template <typename ExecutionPolicy>
+void TestSetDifferenceDevice(ExecutionPolicy exec)
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 2, 4, 5});
+  auto b      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 3, 3, 4, 6});
+  auto result = cuda::make_device_buffer<int>(stream, device, 2, cuda::no_init);
+
+  cuda::launch(stream, test_runtime::single_thread_config(), set_difference_kernel{}, exec, a, b, result);
+  stream.sync();
+
+  test_runtime::assert_equal(stream, result, {2, 5});
+}
+
+void TestSetDifferenceDeviceSeq()
+{
+  TestSetDifferenceDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestSetDifferenceDeviceSeq);
+
+void TestSetDifferenceDeviceDevice()
+{
+  TestSetDifferenceDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestSetDifferenceDeviceDevice);
+#endif
+
+void TestSetDifferenceCudaStreams()
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 2, 4, 5});
+  auto b      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 3, 3, 4, 6});
+  auto result = cuda::make_device_buffer<int>(stream, device, 2, cuda::no_init);
+
+  auto end =
+    thrust::set_difference(thrust::cuda::par.on(stream.get()), a.begin(), a.end(), b.begin(), b.end(), result.begin());
+
+  ASSERT_EQUAL_QUIET(result.end(), end);
+  test_runtime::assert_equal(stream, result, {2, 5});
+}
+DECLARE_UNITTEST(TestSetDifferenceCudaStreams);

cccl_upstream/thrust/testing/cuda/set_difference_by_key.cu

@@ -0,0 +1,118 @@
+#include <thrust/execution_policy.h>
+#include <thrust/set_operations.h>
+#include <thrust/sort.h>
+
+#include <cuda/buffer>
+#include <cuda/cccl_runtime_test_helper.cuh>
+#include <cuda/launch>
+#include <cuda/std/initializer_list>
+#include <cuda/stream>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+struct set_difference_by_key_kernel
+{
+  template <typename ExecutionPolicy,
+            typename Keys1,
+            typename Keys2,
+            typename Values1,
+            typename Values2,
+            typename KeysOutput,
+            typename ValuesOutput>
+  __device__ void operator()(
+    ExecutionPolicy exec,
+    Keys1 keys1,
+    Keys2 keys2,
+    Values1 values1,
+    Values2 values2,
+    KeysOutput keys_result,
+    ValuesOutput values_result) const
+  {
+    auto end = thrust::set_difference_by_key(
+      exec,
+      keys1.begin(),
+      keys1.end(),
+      keys2.begin(),
+      keys2.end(),
+      values1.begin(),
+      values2.begin(),
+      keys_result.begin(),
+      values_result.begin());
+    TEST_ASSERT_DEVICE(end.first == keys_result.end());
+    TEST_ASSERT_DEVICE(end.second == values_result.end());
+  }
+};
+
+template <typename ExecutionPolicy>
+void TestSetDifferenceByKeyDevice(ExecutionPolicy exec)
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a_key      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 2, 4, 5});
+  auto b_key      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 3, 3, 4, 6});
+  auto a_val      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 0, 0, 0});
+  auto b_val      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{1, 1, 1, 1, 1});
+  auto result_key = cuda::make_device_buffer<int>(stream, device, 2, cuda::no_init);
+  auto result_val = cuda::make_device_buffer<int>(stream, device, 2, cuda::no_init);
+
+  cuda::launch(
+    stream,
+    test_runtime::single_thread_config(),
+    set_difference_by_key_kernel{},
+    exec,
+    a_key,
+    b_key,
+    a_val,
+    b_val,
+    result_key,
+    result_val);
+  stream.sync();
+
+  test_runtime::assert_equal(stream, result_key, {2, 5});
+  test_runtime::assert_equal(stream, result_val, {0, 0});
+}
+
+void TestSetDifferenceByKeyDeviceSeq()
+{
+  TestSetDifferenceByKeyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestSetDifferenceByKeyDeviceSeq);
+
+void TestSetDifferenceByKeyDeviceDevice()
+{
+  TestSetDifferenceByKeyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestSetDifferenceByKeyDeviceDevice);
+#endif
+
+void TestSetDifferenceByKeyCudaStreams()
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a_key      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 2, 4, 5});
+  auto b_key      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 3, 3, 4, 6});
+  auto a_val      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 0, 0, 0});
+  auto b_val      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{1, 1, 1, 1, 1});
+  auto result_key = cuda::make_device_buffer<int>(stream, device, 2, cuda::no_init);
+  auto result_val = cuda::make_device_buffer<int>(stream, device, 2, cuda::no_init);
+
+  auto end = thrust::set_difference_by_key(
+    thrust::cuda::par.on(stream.get()),
+    a_key.begin(),
+    a_key.end(),
+    b_key.begin(),
+    b_key.end(),
+    a_val.begin(),
+    b_val.begin(),
+    result_key.begin(),
+    result_val.begin());
+
+  ASSERT_EQUAL_QUIET(result_key.end(), end.first);
+  ASSERT_EQUAL_QUIET(result_val.end(), end.second);
+  test_runtime::assert_equal(stream, result_key, {2, 5});
+  test_runtime::assert_equal(stream, result_val, {0, 0});
+}
+DECLARE_UNITTEST(TestSetDifferenceByKeyCudaStreams);

cccl_upstream/thrust/testing/cuda/set_intersection.cu

@@ -0,0 +1,86 @@
+#include <thrust/execution_policy.h>
+#include <thrust/set_operations.h>
+
+#include <cuda/buffer>
+#include <cuda/cccl_runtime_test_helper.cuh>
+#include <cuda/launch>
+#include <cuda/stream>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+struct set_intersection_kernel
+{
+  template <typename ExecutionPolicy, typename Input1, typename Input2, typename Output>
+  __device__ void operator()(ExecutionPolicy exec, Input1 a, Input2 b, Output result) const
+  {
+    const auto end = thrust::set_intersection(exec, a.begin(), a.end(), b.begin(), b.end(), result.begin());
+    TEST_ASSERT_DEVICE(end == result.end());
+  }
+};
+
+template <typename ExecutionPolicy>
+void TestSetIntersectionDevice(ExecutionPolicy exec)
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a      = cuda::make_device_buffer<int>(stream, device, {0, 2, 4});
+  auto b      = cuda::make_device_buffer<int>(stream, device, {0, 3, 3, 4});
+  auto result = cuda::make_device_buffer<int>(stream, device, 2, cuda::no_init);
+
+  cuda::launch(stream, test_runtime::single_thread_config(), set_intersection_kernel{}, exec, a, b, result);
+  stream.sync();
+
+  test_runtime::assert_equal(stream, result, {0, 4});
+}
+
+void TestSetIntersectionDeviceSeq()
+{
+  TestSetIntersectionDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestSetIntersectionDeviceSeq);
+
+void TestSetIntersectionDeviceDevice()
+{
+  TestSetIntersectionDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestSetIntersectionDeviceDevice);
+
+void TestSetIntersectionDeviceNoSync()
+{
+  TestSetIntersectionDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestSetIntersectionDeviceNoSync);
+#endif
+
+template <typename ExecutionPolicy>
+void TestSetIntersectionCudaStreams(ExecutionPolicy policy)
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a      = cuda::make_device_buffer<int>(stream, device, {0, 2, 4});
+  auto b      = cuda::make_device_buffer<int>(stream, device, {0, 3, 3, 4});
+  auto result = cuda::make_device_buffer<int>(stream, device, 2, cuda::no_init);
+
+  const auto streampolicy = policy.on(stream.get());
+
+  const auto end = thrust::set_intersection(streampolicy, a.begin(), a.end(), b.begin(), b.end(), result.begin());
+  stream.sync();
+
+  ASSERT_EQUAL_QUIET(result.end(), end);
+  test_runtime::assert_equal(stream, result, {0, 4});
+}
+
+void TestSetIntersectionCudaStreamsSync()
+{
+  TestSetIntersectionCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestSetIntersectionCudaStreamsSync);
+
+void TestSetIntersectionCudaStreamsNoSync()
+{
+  TestSetIntersectionCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestSetIntersectionCudaStreamsNoSync);

cccl_upstream/thrust/testing/cuda/set_intersection_by_key.cu

@@ -0,0 +1,126 @@
+#include <thrust/execution_policy.h>
+#include <thrust/set_operations.h>
+
+#include <cuda/buffer>
+#include <cuda/cccl_runtime_test_helper.cuh>
+#include <cuda/launch>
+#include <cuda/stream>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+struct set_intersection_by_key_kernel
+{
+  template <typename ExecutionPolicy,
+            typename Keys1,
+            typename Keys2,
+            typename Values1,
+            typename KeysOutput,
+            typename ValuesOutput>
+  __device__ void operator()(
+    ExecutionPolicy exec, Keys1 keys1, Keys2 keys2, Values1 values1, KeysOutput keys_result, ValuesOutput values_result)
+    const
+  {
+    auto end = thrust::set_intersection_by_key(
+      exec,
+      keys1.begin(),
+      keys1.end(),
+      keys2.begin(),
+      keys2.end(),
+      values1.begin(),
+      keys_result.begin(),
+      values_result.begin());
+    TEST_ASSERT_DEVICE(end.first == keys_result.end());
+    TEST_ASSERT_DEVICE(end.second == values_result.end());
+  }
+};
+
+template <typename ExecutionPolicy>
+void TestSetIntersectionByKeyDevice(ExecutionPolicy exec)
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a_key      = cuda::make_device_buffer<int>(stream, device, {0, 2, 4});
+  auto b_key      = cuda::make_device_buffer<int>(stream, device, {0, 3, 3, 4});
+  auto a_val      = cuda::make_device_buffer<int>(stream, device, {0, 0, 0});
+  auto result_key = cuda::make_device_buffer<int>(stream, device, 2, cuda::no_init);
+  auto result_val = cuda::make_device_buffer<int>(stream, device, 2, cuda::no_init);
+
+  cuda::launch(
+    stream,
+    test_runtime::single_thread_config(),
+    set_intersection_by_key_kernel{},
+    exec,
+    a_key,
+    b_key,
+    a_val,
+    result_key,
+    result_val);
+  stream.sync();
+
+  test_runtime::assert_equal(stream, result_key, {0, 4});
+  test_runtime::assert_equal(stream, result_val, {0, 0});
+}
+
+void TestSetIntersectionByKeyDeviceSeq()
+{
+  TestSetIntersectionByKeyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestSetIntersectionByKeyDeviceSeq);
+
+void TestSetIntersectionByKeyDeviceDevice()
+{
+  TestSetIntersectionByKeyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestSetIntersectionByKeyDeviceDevice);
+
+void TestSetIntersectionByKeyDeviceNoSync()
+{
+  TestSetIntersectionByKeyDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestSetIntersectionByKeyDeviceNoSync);
+#endif
+
+template <typename ExecutionPolicy>
+void TestSetIntersectionByKeyCudaStreams(ExecutionPolicy policy)
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a_key      = cuda::make_device_buffer<int>(stream, device, {0, 2, 4});
+  auto b_key      = cuda::make_device_buffer<int>(stream, device, {0, 3, 3, 4});
+  auto a_val      = cuda::make_device_buffer<int>(stream, device, {0, 0, 0});
+  auto result_key = cuda::make_device_buffer<int>(stream, device, 2, cuda::no_init);
+  auto result_val = cuda::make_device_buffer<int>(stream, device, 2, cuda::no_init);
+
+  const auto streampolicy = policy.on(stream.get());
+
+  const auto end = thrust::set_intersection_by_key(
+    streampolicy,
+    a_key.begin(),
+    a_key.end(),
+    b_key.begin(),
+    b_key.end(),
+    a_val.begin(),
+    result_key.begin(),
+    result_val.begin());
+  stream.sync();
+
+  ASSERT_EQUAL_QUIET(result_key.end(), end.first);
+  ASSERT_EQUAL_QUIET(result_val.end(), end.second);
+  test_runtime::assert_equal(stream, result_key, {0, 4});
+  test_runtime::assert_equal(stream, result_val, {0, 0});
+}
+
+void TestSetIntersectionByKeyCudaStreamsSync()
+{
+  TestSetIntersectionByKeyCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestSetIntersectionByKeyCudaStreamsSync);
+
+void TestSetIntersectionByKeyCudaStreamsNoSync()
+{
+  TestSetIntersectionByKeyCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestSetIntersectionByKeyCudaStreamsNoSync);

cccl_upstream/thrust/testing/cuda/set_symmetric_difference.cu

@@ -0,0 +1,67 @@
+#include <thrust/execution_policy.h>
+#include <thrust/set_operations.h>
+
+#include <cuda/buffer>
+#include <cuda/cccl_runtime_test_helper.cuh>
+#include <cuda/launch>
+#include <cuda/std/initializer_list>
+#include <cuda/stream>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+struct set_symmetric_difference_kernel
+{
+  template <typename ExecutionPolicy, typename Input1, typename Input2, typename Output>
+  __device__ void operator()(ExecutionPolicy exec, Input1 a, Input2 b, Output result) const
+  {
+    const auto end = thrust::set_symmetric_difference(exec, a.begin(), a.end(), b.begin(), b.end(), result.begin());
+    TEST_ASSERT_DEVICE(end == result.end());
+  }
+};
+
+template <typename ExecutionPolicy>
+void TestSetSymmetricDifferenceDevice(ExecutionPolicy exec)
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 2, 4, 6});
+  auto b      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 3, 3, 4, 7});
+  auto result = cuda::make_device_buffer<int>(stream, device, 5, cuda::no_init);
+
+  cuda::launch(stream, test_runtime::single_thread_config(), set_symmetric_difference_kernel{}, exec, a, b, result);
+  stream.sync();
+
+  test_runtime::assert_equal(stream, result, {2, 3, 3, 6, 7});
+}
+
+void TestSetSymmetricDifferenceDeviceSeq()
+{
+  TestSetSymmetricDifferenceDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestSetSymmetricDifferenceDeviceSeq);
+
+void TestSetSymmetricDifferenceDeviceDevice()
+{
+  TestSetSymmetricDifferenceDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestSetSymmetricDifferenceDeviceDevice);
+#endif
+
+void TestSetSymmetricDifferenceCudaStreams()
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 2, 4, 6});
+  auto b      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 3, 3, 4, 7});
+  auto result = cuda::make_device_buffer<int>(stream, device, 5, cuda::no_init);
+
+  auto end = thrust::set_symmetric_difference(
+    thrust::cuda::par.on(stream.get()), a.begin(), a.end(), b.begin(), b.end(), result.begin());
+
+  ASSERT_EQUAL_QUIET(result.end(), end);
+  test_runtime::assert_equal(stream, result, {2, 3, 3, 6, 7});
+}
+DECLARE_UNITTEST(TestSetSymmetricDifferenceCudaStreams);

cccl_upstream/thrust/testing/cuda/set_symmetric_difference_by_key.cu

@@ -0,0 +1,117 @@
+#include <thrust/execution_policy.h>
+#include <thrust/set_operations.h>
+
+#include <cuda/buffer>
+#include <cuda/cccl_runtime_test_helper.cuh>
+#include <cuda/launch>
+#include <cuda/std/initializer_list>
+#include <cuda/stream>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+struct set_symmetric_difference_by_key_kernel
+{
+  template <typename ExecutionPolicy,
+            typename Keys1,
+            typename Keys2,
+            typename Values1,
+            typename Values2,
+            typename KeysOutput,
+            typename ValuesOutput>
+  __device__ void operator()(
+    ExecutionPolicy exec,
+    Keys1 keys1,
+    Keys2 keys2,
+    Values1 values1,
+    Values2 values2,
+    KeysOutput keys_result,
+    ValuesOutput values_result) const
+  {
+    auto end = thrust::set_symmetric_difference_by_key(
+      exec,
+      keys1.begin(),
+      keys1.end(),
+      keys2.begin(),
+      keys2.end(),
+      values1.begin(),
+      values2.begin(),
+      keys_result.begin(),
+      values_result.begin());
+    TEST_ASSERT_DEVICE(end.first == keys_result.end());
+    TEST_ASSERT_DEVICE(end.second == values_result.end());
+  }
+};
+
+template <typename ExecutionPolicy>
+void TestSetSymmetricDifferenceByKeyDevice(ExecutionPolicy exec)
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a_key      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 2, 4, 6});
+  auto b_key      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 3, 3, 4, 7});
+  auto a_val      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 0, 0, 0});
+  auto b_val      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{1, 1, 1, 1, 1});
+  auto result_key = cuda::make_device_buffer<int>(stream, device, 5, cuda::no_init);
+  auto result_val = cuda::make_device_buffer<int>(stream, device, 5, cuda::no_init);
+
+  cuda::launch(
+    stream,
+    test_runtime::single_thread_config(),
+    set_symmetric_difference_by_key_kernel{},
+    exec,
+    a_key,
+    b_key,
+    a_val,
+    b_val,
+    result_key,
+    result_val);
+  stream.sync();
+
+  test_runtime::assert_equal(stream, result_key, {2, 3, 3, 6, 7});
+  test_runtime::assert_equal(stream, result_val, {0, 1, 1, 0, 1});
+}
+
+void TestSetSymmetricDifferenceByKeyDeviceSeq()
+{
+  TestSetSymmetricDifferenceByKeyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestSetSymmetricDifferenceByKeyDeviceSeq);
+
+void TestSetSymmetricDifferenceByKeyDeviceDevice()
+{
+  TestSetSymmetricDifferenceByKeyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestSetSymmetricDifferenceByKeyDeviceDevice);
+#endif
+
+void TestSetSymmetricDifferenceByKeyCudaStreams()
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a_key      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 2, 4, 6});
+  auto b_key      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 3, 3, 4, 7});
+  auto a_val      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 0, 0, 0});
+  auto b_val      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{1, 1, 1, 1, 1});
+  auto result_key = cuda::make_device_buffer<int>(stream, device, 5, cuda::no_init);
+  auto result_val = cuda::make_device_buffer<int>(stream, device, 5, cuda::no_init);
+
+  auto end = thrust::set_symmetric_difference_by_key(
+    thrust::cuda::par.on(stream.get()),
+    a_key.begin(),
+    a_key.end(),
+    b_key.begin(),
+    b_key.end(),
+    a_val.begin(),
+    b_val.begin(),
+    result_key.begin(),
+    result_val.begin());
+
+  ASSERT_EQUAL_QUIET(result_key.end(), end.first);
+  ASSERT_EQUAL_QUIET(result_val.end(), end.second);
+  test_runtime::assert_equal(stream, result_key, {2, 3, 3, 6, 7});
+  test_runtime::assert_equal(stream, result_val, {0, 1, 1, 0, 1});
+}
+DECLARE_UNITTEST(TestSetSymmetricDifferenceByKeyCudaStreams);

cccl_upstream/thrust/testing/cuda/set_union.cu

@@ -0,0 +1,67 @@
+#include <thrust/execution_policy.h>
+#include <thrust/set_operations.h>
+
+#include <cuda/buffer>
+#include <cuda/cccl_runtime_test_helper.cuh>
+#include <cuda/launch>
+#include <cuda/std/initializer_list>
+#include <cuda/stream>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+struct set_union_kernel
+{
+  template <typename ExecutionPolicy, typename Input1, typename Input2, typename Output>
+  __device__ void operator()(ExecutionPolicy exec, Input1 a, Input2 b, Output result) const
+  {
+    const auto end = thrust::set_union(exec, a.begin(), a.end(), b.begin(), b.end(), result.begin());
+    TEST_ASSERT_DEVICE(end == result.end());
+  }
+};
+
+template <typename ExecutionPolicy>
+void TestSetUnionDevice(ExecutionPolicy exec)
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 2, 4});
+  auto b      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 3, 3, 4});
+  auto result = cuda::make_device_buffer<int>(stream, device, 5, cuda::no_init);
+
+  cuda::launch(stream, test_runtime::single_thread_config(), set_union_kernel{}, exec, a, b, result);
+  stream.sync();
+
+  test_runtime::assert_equal(stream, result, {0, 2, 3, 3, 4});
+}
+
+void TestSetUnionDeviceSeq()
+{
+  TestSetUnionDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestSetUnionDeviceSeq);
+
+void TestSetUnionDeviceDevice()
+{
+  TestSetUnionDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestSetUnionDeviceDevice);
+#endif
+
+void TestSetUnionCudaStreams()
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 2, 4});
+  auto b      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 3, 3, 4});
+  auto result = cuda::make_device_buffer<int>(stream, device, 5, cuda::no_init);
+
+  auto end =
+    thrust::set_union(thrust::cuda::par.on(stream.get()), a.begin(), a.end(), b.begin(), b.end(), result.begin());
+
+  ASSERT_EQUAL_QUIET(result.end(), end);
+  test_runtime::assert_equal(stream, result, {0, 2, 3, 3, 4});
+}
+DECLARE_UNITTEST(TestSetUnionCudaStreams);

cccl_upstream/thrust/testing/cuda/set_union_by_key.cu

@@ -0,0 +1,117 @@
+#include <thrust/execution_policy.h>
+#include <thrust/set_operations.h>
+
+#include <cuda/buffer>
+#include <cuda/cccl_runtime_test_helper.cuh>
+#include <cuda/launch>
+#include <cuda/std/initializer_list>
+#include <cuda/stream>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+struct set_union_by_key_kernel
+{
+  template <typename ExecutionPolicy,
+            typename Keys1,
+            typename Keys2,
+            typename Values1,
+            typename Values2,
+            typename KeysOutput,
+            typename ValuesOutput>
+  __device__ void operator()(
+    ExecutionPolicy exec,
+    Keys1 keys1,
+    Keys2 keys2,
+    Values1 values1,
+    Values2 values2,
+    KeysOutput keys_result,
+    ValuesOutput values_result) const
+  {
+    auto end = thrust::set_union_by_key(
+      exec,
+      keys1.begin(),
+      keys1.end(),
+      keys2.begin(),
+      keys2.end(),
+      values1.begin(),
+      values2.begin(),
+      keys_result.begin(),
+      values_result.begin());
+    TEST_ASSERT_DEVICE(end.first == keys_result.end());
+    TEST_ASSERT_DEVICE(end.second == values_result.end());
+  }
+};
+
+template <typename ExecutionPolicy>
+void TestSetUnionByKeyDevice(ExecutionPolicy exec)
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a_key      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 2, 4});
+  auto b_key      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 3, 3, 4});
+  auto a_val      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 0, 0});
+  auto b_val      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{1, 1, 1, 1});
+  auto result_key = cuda::make_device_buffer<int>(stream, device, 5, cuda::no_init);
+  auto result_val = cuda::make_device_buffer<int>(stream, device, 5, cuda::no_init);
+
+  cuda::launch(
+    stream,
+    test_runtime::single_thread_config(),
+    set_union_by_key_kernel{},
+    exec,
+    a_key,
+    b_key,
+    a_val,
+    b_val,
+    result_key,
+    result_val);
+  stream.sync();
+
+  test_runtime::assert_equal(stream, result_key, {0, 2, 3, 3, 4});
+  test_runtime::assert_equal(stream, result_val, {0, 0, 1, 1, 0});
+}
+
+void TestSetUnionByKeyDeviceSeq()
+{
+  TestSetUnionByKeyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestSetUnionByKeyDeviceSeq);
+
+void TestSetUnionByKeyDeviceDevice()
+{
+  TestSetUnionByKeyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestSetUnionByKeyDeviceDevice);
+#endif
+
+void TestSetUnionByKeyCudaStreams()
+{
+  const auto device = test_runtime::current_test_device();
+  cuda::stream stream{device};
+
+  auto a_key      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 2, 4});
+  auto b_key      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 3, 3, 4});
+  auto a_val      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{0, 0, 0});
+  auto b_val      = cuda::make_device_buffer<int>(stream, device, cuda::std::initializer_list<int>{1, 1, 1, 1});
+  auto result_key = cuda::make_device_buffer<int>(stream, device, 5, cuda::no_init);
+  auto result_val = cuda::make_device_buffer<int>(stream, device, 5, cuda::no_init);
+
+  auto end = thrust::set_union_by_key(
+    thrust::cuda::par.on(stream.get()),
+    a_key.begin(),
+    a_key.end(),
+    b_key.begin(),
+    b_key.end(),
+    a_val.begin(),
+    b_val.begin(),
+    result_key.begin(),
+    result_val.begin());
+
+  ASSERT_EQUAL_QUIET(result_key.end(), end.first);
+  ASSERT_EQUAL_QUIET(result_val.end(), end.second);
+  test_runtime::assert_equal(stream, result_key, {0, 2, 3, 3, 4});
+  test_runtime::assert_equal(stream, result_val, {0, 0, 1, 1, 0});
+}
+DECLARE_UNITTEST(TestSetUnionByKeyCudaStreams);

cccl_upstream/thrust/testing/cuda/sort.cu

@@ -0,0 +1,333 @@
+#include <thrust/copy.h>
+#include <thrust/equal.h>
+#include <thrust/execution_policy.h>
+#include <thrust/functional.h>
+#include <thrust/iterator/counting_iterator.h>
+#include <thrust/iterator/transform_iterator.h>
+#include <thrust/sort.h>
+
+#include <cuda/std/iterator>
+#include <cuda/std/limits>
+
+#include <algorithm>
+#include <cstdint>
+#include <exception>
+
+#include <unittest/unittest.h>
+
+template <typename T>
+struct my_less
+{
+  _CCCL_HOST_DEVICE bool operator()(const T& lhs, const T& rhs) const
+  {
+    return lhs < rhs;
+  }
+};
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename Compare>
+__global__ void sort_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Compare comp)
+{
+  thrust::sort(exec, first, last, comp);
+}
+
+template <typename T, typename ExecutionPolicy, typename Compare>
+void TestComparisonSortDevice(ExecutionPolicy exec, const size_t n, Compare comp)
+{
+  thrust::host_vector<T> h_data   = unittest::random_integers<T>(n);
+  thrust::device_vector<T> d_data = h_data;
+
+  sort_kernel<<<1, 1>>>(exec, d_data.begin(), d_data.end(), comp);
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  thrust::sort(h_data.begin(), h_data.end(), comp);
+
+  ASSERT_EQUAL(h_data, d_data);
+};
+
+template <typename T>
+struct TestComparisonSortDeviceSeq
+{
+  void operator()(const size_t n)
+  {
+    TestComparisonSortDevice<T>(thrust::seq, n, my_less<T>());
+  }
+};
+VariableUnitTest<TestComparisonSortDeviceSeq, unittest::type_list<unittest::int8_t, unittest::int32_t>>
+  TestComparisonSortDeviceSeqInstance;
+
+template <typename T>
+struct TestComparisonSortDeviceDevice
+{
+  void operator()(const size_t n)
+  {
+    TestComparisonSortDevice<T>(thrust::device, n, my_less<T>());
+  }
+};
+VariableUnitTest<TestComparisonSortDeviceDevice, unittest::type_list<unittest::int8_t, unittest::int32_t>>
+  TestComparisonSortDeviceDeviceDeviceInstance;
+
+template <typename T, typename ExecutionPolicy>
+void TestSortDevice(ExecutionPolicy exec, const size_t n)
+{
+  TestComparisonSortDevice<T>(exec, n, ::cuda::std::less<T>());
+};
+
+template <typename T>
+struct TestSortDeviceSeq
+{
+  void operator()(const size_t n)
+  {
+    TestSortDevice<T>(thrust::seq, n);
+  }
+};
+VariableUnitTest<TestSortDeviceSeq, unittest::type_list<unittest::int8_t, unittest::int32_t>> TestSortDeviceSeqInstance;
+
+template <typename T>
+struct TestSortDeviceDevice
+{
+  void operator()(const size_t n)
+  {
+    TestSortDevice<T>(thrust::device, n);
+  }
+};
+VariableUnitTest<TestSortDeviceDevice, unittest::type_list<unittest::int8_t, unittest::int32_t>>
+  TestSortDeviceDeviceInstance;
+#endif
+
+void TestSortCudaStreams()
+{
+  thrust::device_vector<int> keys{9, 3, 2, 0, 4, 7, 8, 1, 5, 6};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::sort(thrust::cuda::par.on(s), keys.begin(), keys.end());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(true, thrust::is_sorted(keys.begin(), keys.end()));
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestSortCudaStreams);
+
+void TestComparisonSortCudaStreams()
+{
+  thrust::device_vector<int> keys{9, 3, 2, 0, 4, 7, 8, 1, 5, 6};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::sort(thrust::cuda::par.on(s), keys.begin(), keys.end(), my_less<int>());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(true, thrust::is_sorted(keys.begin(), keys.end(), my_less<int>()));
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestComparisonSortCudaStreams);
+
+template <typename T>
+struct TestRadixSortDispatch
+{
+  static_assert(cub::__can_use_radix_sort<T*, ::cuda::std::less<T>>);
+  static_assert(cub::__can_use_radix_sort<T*, ::cuda::std::greater<T>>);
+  static_assert(cub::__can_use_radix_sort<T*, ::cuda::std::less<T>>);
+  static_assert(cub::__can_use_radix_sort<T*, ::cuda::std::greater<T>>);
+
+  static_assert(cub::__can_use_radix_sort<T*, ::cuda::std::less<>>);
+  static_assert(cub::__can_use_radix_sort<T*, ::cuda::std::greater<>>);
+  static_assert(cub::__can_use_radix_sort<T*, ::cuda::std::less<>>);
+  static_assert(cub::__can_use_radix_sort<T*, ::cuda::std::greater<>>);
+
+  static_assert(cub::__can_use_radix_sort<const T*, ::cuda::std::less<T>>);
+  static_assert(cub::__can_use_radix_sort<const T*, ::cuda::std::greater<T>>);
+  static_assert(cub::__can_use_radix_sort<const T*, ::cuda::std::less<T>>);
+  static_assert(cub::__can_use_radix_sort<const T*, ::cuda::std::greater<T>>);
+
+  static_assert(cub::__can_use_radix_sort<const T*, ::cuda::std::less<>>);
+  static_assert(cub::__can_use_radix_sort<const T*, ::cuda::std::greater<>>);
+  static_assert(cub::__can_use_radix_sort<const T*, ::cuda::std::less<>>);
+  static_assert(cub::__can_use_radix_sort<const T*, ::cuda::std::greater<>>);
+
+  static_assert(cub::__can_use_radix_sort<T*, const ::cuda::std::less<T>>);
+  static_assert(cub::__can_use_radix_sort<T*, const ::cuda::std::greater<T>>);
+  static_assert(cub::__can_use_radix_sort<T*, const ::cuda::std::less<T>>);
+  static_assert(cub::__can_use_radix_sort<T*, const ::cuda::std::greater<T>>);
+
+  static_assert(cub::__can_use_radix_sort<T*, const ::cuda::std::less<>>);
+  static_assert(cub::__can_use_radix_sort<T*, const ::cuda::std::greater<>>);
+  static_assert(cub::__can_use_radix_sort<T*, const ::cuda::std::less<>>);
+  static_assert(cub::__can_use_radix_sort<T*, const ::cuda::std::greater<>>);
+
+  static_assert(cub::__can_use_radix_sort<cuda::std::reverse_iterator<T*>, ::cuda::std::less<T>>);
+  static_assert(cub::__can_use_radix_sort<cuda::std::reverse_iterator<T*>, ::cuda::std::greater<T>>);
+  static_assert(cub::__can_use_radix_sort<cuda::std::reverse_iterator<T*>, ::cuda::std::less<T>>);
+  static_assert(cub::__can_use_radix_sort<cuda::std::reverse_iterator<T*>, ::cuda::std::greater<T>>);
+
+  static_assert(cub::__can_use_radix_sort<cuda::std::reverse_iterator<T*>, ::cuda::std::less<>>);
+  static_assert(cub::__can_use_radix_sort<cuda::std::reverse_iterator<T*>, ::cuda::std::greater<>>);
+  static_assert(cub::__can_use_radix_sort<cuda::std::reverse_iterator<T*>, ::cuda::std::less<>>);
+  static_assert(cub::__can_use_radix_sort<cuda::std::reverse_iterator<T*>, ::cuda::std::greater<>>);
+
+  void operator()() const {}
+};
+SimpleUnitTest<TestRadixSortDispatch,
+               unittest::concat<IntegralTypes,
+                                FloatingPointTypes
+#if _CCCL_HAS_INT128()
+                                ,
+                                unittest::type_list<__int128_t, __uint128_t>
+#endif // _CCCL_HAS_INT128()
+#if _CCCL_HAS_NVFP16()
+                                ,
+                                unittest::type_list<__half>
+#endif // _CCCL_HAS_NVFP16()
+#if _CCCL_HAS_NVBF16()
+                                ,
+                                unittest::type_list<__nv_bfloat16>
+#endif // _CCCL_HAS_NVBF16()
+                                >>
+  TestRadixSortDispatchInstance;
+
+/**
+ * Copy of CUB testing utility
+ */
+template <typename UnsignedIntegralKeyT>
+struct index_to_key_value_op
+{
+  static constexpr std::size_t max_key_value =
+    static_cast<std::size_t>(::cuda::std::numeric_limits<UnsignedIntegralKeyT>::max());
+  static constexpr std::size_t lowest_key_value =
+    static_cast<std::size_t>(::cuda::std::numeric_limits<UnsignedIntegralKeyT>::lowest());
+  static_assert(sizeof(UnsignedIntegralKeyT) < sizeof(std::size_t),
+                "Calculation of num_distinct_key_values would overflow");
+  static constexpr std::size_t num_distinct_key_values = (max_key_value - lowest_key_value + std::size_t{1ULL});
+
+  __device__ __host__ UnsignedIntegralKeyT operator()(std::size_t index)
+  {
+    return static_cast<UnsignedIntegralKeyT>(index % num_distinct_key_values);
+  }
+};
+
+/**
+ * Copy of CUB testing utility
+ */
+template <typename UnsignedIntegralKeyT>
+class index_to_expected_key_op
+{
+private:
+  static constexpr std::size_t max_key_value =
+    static_cast<std::size_t>(::cuda::std::numeric_limits<UnsignedIntegralKeyT>::max());
+  static constexpr std::size_t lowest_key_value =
+    static_cast<std::size_t>(::cuda::std::numeric_limits<UnsignedIntegralKeyT>::lowest());
+  static_assert(sizeof(UnsignedIntegralKeyT) < sizeof(std::size_t),
+                "Calculation of num_distinct_key_values would overflow");
+  static constexpr std::size_t num_distinct_key_values = (max_key_value - lowest_key_value + std::size_t{1ULL});
+
+  // item_count / num_distinct_key_values
+  std::size_t expected_count_per_item;
+  // num remainder items: item_count%num_distinct_key_values
+  std::size_t num_remainder_items;
+  // remainder item_count: expected_count_per_item+1
+  std::size_t remainder_item_count;
+
+public:
+  index_to_expected_key_op(std::size_t num_total_items)
+      : expected_count_per_item(num_total_items / num_distinct_key_values)
+      , num_remainder_items(num_total_items % num_distinct_key_values)
+      , remainder_item_count(expected_count_per_item + std::size_t{1ULL})
+  {}
+
+  __device__ __host__ UnsignedIntegralKeyT operator()(std::size_t index)
+  {
+    // The first (num_remainder_items * remainder_item_count) are items that appear once more often than the items that
+    // follow remainder_items_offset
+    std::size_t remainder_items_offset = num_remainder_items * remainder_item_count;
+
+    UnsignedIntegralKeyT target_item_index =
+      (index <= remainder_items_offset)
+        ?
+        // This is one of the remainder items
+        static_cast<UnsignedIntegralKeyT>(index / remainder_item_count)
+        :
+        // This is an item that appears exactly expected_count_per_item times
+        static_cast<UnsignedIntegralKeyT>(
+          num_remainder_items + ((index - remainder_items_offset) / expected_count_per_item));
+    return target_item_index;
+  }
+};
+
+void TestSortWithMagnitude(int magnitude)
+{
+  try
+  {
+    const std::size_t num_items = 1ull << magnitude;
+    thrust::device_vector<std::uint8_t> vec(num_items);
+    auto counting_it   = thrust::make_counting_iterator(std::size_t{0});
+    auto key_value_it  = thrust::make_transform_iterator(counting_it, index_to_key_value_op<std::uint8_t>{});
+    auto rev_sorted_it = cuda::std::make_reverse_iterator(key_value_it + static_cast<std::ptrdiff_t>(num_items));
+    thrust::copy(rev_sorted_it, rev_sorted_it + static_cast<std::ptrdiff_t>(num_items), vec.begin());
+    thrust::sort(vec.begin(), vec.end());
+    auto expected_result_it = thrust::make_transform_iterator(
+      thrust::make_counting_iterator(std::size_t{}), index_to_expected_key_op<std::uint8_t>(num_items));
+    const bool ok =
+      thrust::equal(expected_result_it, expected_result_it + static_cast<std::ptrdiff_t>(num_items), vec.cbegin());
+    ASSERT_EQUAL(ok, true);
+  }
+  catch (std::bad_alloc&)
+  {
+    return;
+  }
+}
+
+void TestSortWithLargeNumberOfItems()
+{
+  TestSortWithMagnitude(30);
+  // These still require 64-bit dispatches when magnitude < 32.
+#ifndef THRUST_FORCE_32_BIT_OFFSET_TYPE
+  TestSortWithMagnitude(31);
+  TestSortWithMagnitude(32);
+  TestSortWithMagnitude(33);
+  TestSortWithMagnitude(39);
+#endif
+}
+DECLARE_UNITTEST(TestSortWithLargeNumberOfItems);
+
+template <typename T>
+struct TestSortAscendingKey
+{
+  void operator()() const
+  {
+    constexpr int n = 10000;
+
+    thrust::host_vector<T> h_data   = unittest::random_integers<T>(n);
+    thrust::device_vector<T> d_data = h_data;
+
+    std::sort(h_data.begin(), h_data.end(), ::cuda::std::less<T>{});
+    thrust::sort(d_data.begin(), d_data.end(), ::cuda::std::less<T>{});
+
+    ASSERT_EQUAL_QUIET(h_data, d_data);
+  }
+};
+
+SimpleUnitTest<TestSortAscendingKey,
+               unittest::concat<unittest::type_list<>
+#if _CCCL_HAS_INT128()
+                                ,
+                                unittest::type_list<__int128_t, __uint128_t>
+#endif // _CCCL_HAS_INT128()
+// CTK 12.2 offers __host__ __device__ operators for __half and __nv_bfloat16, so we can use std::sort
+#if _CCCL_CTK_AT_LEAST(12, 2)
+#  if _CCCL_HAS_NVFP16() || !defined(__CUDA_NO_HALF_OPERATORS__) && !defined(__CUDA_NO_HALF_CONVERSIONS__)
+                                ,
+                                unittest::type_list<__half>
+#  endif
+#  if _CCCL_HAS_NVBF16() || !defined(__CUDA_NO_BFLOAT16_OPERATORS__) && !defined(__CUDA_NO_BFLOAT16_CONVERSIONS__)
+                                ,
+                                unittest::type_list<__nv_bfloat16>
+#  endif
+#endif // _CCCL_CTK_AT_LEAST(12, 2)
+                                >>
+  TestSortAscendingKeyMoreTypes;

cccl_upstream/thrust/testing/cuda/sort_by_key.cu

@@ -0,0 +1,128 @@
+#include <thrust/execution_policy.h>
+#include <thrust/functional.h>
+#include <thrust/sort.h>
+
+#include <unittest/unittest.h>
+
+template <typename T>
+struct my_less
+{
+  _CCCL_HOST_DEVICE bool operator()(const T& lhs, const T& rhs) const
+  {
+    return lhs < rhs;
+  }
+};
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Compare>
+__global__ void sort_by_key_kernel(
+  ExecutionPolicy exec, Iterator1 keys_first, Iterator1 keys_last, Iterator2 values_first, Compare comp)
+{
+  thrust::sort_by_key(exec, keys_first, keys_last, values_first, comp);
+}
+
+template <typename T, typename ExecutionPolicy, typename Compare>
+void TestComparisonSortByKeyDevice(ExecutionPolicy exec, const size_t n, Compare comp)
+{
+  thrust::host_vector<T> h_keys   = unittest::random_integers<T>(n);
+  thrust::device_vector<T> d_keys = h_keys;
+
+  thrust::host_vector<T> h_values   = h_keys;
+  thrust::device_vector<T> d_values = d_keys;
+
+  sort_by_key_kernel<<<1, 1>>>(exec, d_keys.begin(), d_keys.end(), d_values.begin(), comp);
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  thrust::sort_by_key(h_keys.begin(), h_keys.end(), h_values.begin(), comp);
+
+  ASSERT_EQUAL(h_keys, d_keys);
+  ASSERT_EQUAL(h_values, d_values);
+};
+
+template <typename T>
+struct TestComparisonSortByKeyDeviceSeq
+{
+  void operator()(const size_t n)
+  {
+    TestComparisonSortByKeyDevice<T>(thrust::seq, n, my_less<T>());
+  }
+};
+VariableUnitTest<TestComparisonSortByKeyDeviceSeq, unittest::type_list<unittest::int8_t, unittest::int32_t>>
+  TestComparisonSortByKeyDeviceSeqInstance;
+
+template <typename T>
+struct TestComparisonSortByKeyDeviceDevice
+{
+  void operator()(const size_t n)
+  {
+    TestComparisonSortByKeyDevice<T>(thrust::device, n, my_less<T>());
+  }
+};
+VariableUnitTest<TestComparisonSortByKeyDeviceDevice, unittest::type_list<unittest::int8_t, unittest::int32_t>>
+  TestComparisonSortByKeyDeviceDeviceDeviceInstance;
+
+template <typename T, typename ExecutionPolicy>
+void TestSortByKeyDevice(ExecutionPolicy exec, const size_t n)
+{
+  TestComparisonSortByKeyDevice<T>(exec, n, ::cuda::std::less<T>());
+};
+
+template <typename T>
+struct TestSortByKeyDeviceSeq
+{
+  void operator()(const size_t n)
+  {
+    TestSortByKeyDevice<T>(thrust::seq, n);
+  }
+};
+VariableUnitTest<TestSortByKeyDeviceSeq, unittest::type_list<unittest::int8_t, unittest::int32_t>>
+  TestSortByKeyDeviceSeqInstance;
+
+template <typename T>
+struct TestSortByKeyDeviceDevice
+{
+  void operator()(const size_t n)
+  {
+    TestSortByKeyDevice<T>(thrust::device, n);
+  }
+};
+VariableUnitTest<TestSortByKeyDeviceDevice, unittest::type_list<unittest::int8_t, unittest::int32_t>>
+  TestSortByKeyDeviceDeviceInstance;
+#endif
+
+void TestComparisonSortByKeyCudaStreams()
+{
+  thrust::device_vector<int> keys{9, 3, 2, 0, 4, 7, 8, 1, 5, 6};
+  thrust::device_vector<int> vals{9, 3, 2, 0, 4, 7, 8, 1, 5, 6};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::sort_by_key(thrust::cuda::par.on(s), keys.begin(), keys.end(), vals.begin(), my_less<int>());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(true, thrust::is_sorted(keys.begin(), keys.end()));
+  ASSERT_EQUAL(true, thrust::is_sorted(vals.begin(), vals.end()));
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestComparisonSortByKeyCudaStreams);
+
+void TestSortByKeyCudaStreams()
+{
+  thrust::device_vector<int> keys{9, 3, 2, 0, 4, 7, 8, 1, 5, 6};
+  thrust::device_vector<int> vals{9, 3, 2, 0, 4, 7, 8, 1, 5, 6};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::sort_by_key(thrust::cuda::par.on(s), keys.begin(), keys.end(), vals.begin());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(true, thrust::is_sorted(keys.begin(), keys.end()));
+  ASSERT_EQUAL(true, thrust::is_sorted(vals.begin(), vals.end()));
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestSortByKeyCudaStreams);

cccl_upstream/thrust/testing/cuda/stream_legacy.cu

@@ -0,0 +1,22 @@
+#include <thrust/execution_policy.h>
+#include <thrust/system/cuda/detail/util.h>
+
+#include <thread>
+
+#include <unittest/unittest.h>
+
+void verify_stream()
+{
+  auto exec   = thrust::device;
+  auto stream = thrust::cuda_cub::stream(exec);
+  ASSERT_EQUAL(stream, cudaStreamLegacy);
+}
+
+void TestLegacyDefaultStream()
+{
+  verify_stream();
+
+  std::thread t(verify_stream);
+  t.join();
+}
+DECLARE_UNITTEST(TestLegacyDefaultStream);

cccl_upstream/thrust/testing/cuda/stream_per_thread.cmake

@@ -0,0 +1,13 @@
+# This test should always use per-thread streams on NVCC.
+set_target_properties(
+  ${test_target}
+  PROPERTIES
+    COMPILE_OPTIONS
+      $<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--default-stream=per-thread>
+)
+
+# NVC++ does not have an equivalent option, and will always
+# use the global stream by default.
+if (CMAKE_CUDA_COMPILER_ID STREQUAL "NVHPC")
+  set_tests_properties(${test_target} PROPERTIES WILL_FAIL ON)
+endif()

cccl_upstream/thrust/testing/cuda/stream_per_thread.cu

@@ -0,0 +1,22 @@
+#include <thrust/execution_policy.h>
+#include <thrust/system/cuda/detail/util.h>
+
+#include <thread>
+
+#include <unittest/unittest.h>
+
+void verify_stream()
+{
+  auto exec   = thrust::device;
+  auto stream = thrust::cuda_cub::stream(exec);
+  ASSERT_EQUAL(stream, cudaStreamPerThread);
+}
+
+void TestPerThreadDefaultStream()
+{
+  verify_stream();
+
+  std::thread t(verify_stream);
+  t.join();
+}
+DECLARE_UNITTEST(TestPerThreadDefaultStream);

cccl_upstream/thrust/testing/cuda/stream_wrapper.cu

@@ -0,0 +1,86 @@
+#include <thrust/execution_policy.h>
+#include <thrust/reduce.h>
+
+#include <cuda/stream>
+
+#include <unittest/unittest.h>
+
+// Simple non-owning stream wrapper that allows implicit conversion to cudaStream_t.
+struct stream_wrapper
+{
+  stream_wrapper(cudaStream_t s)
+      : stream(s)
+  {}
+
+  operator cudaStream_t() const
+  {
+    return stream;
+  }
+
+  cudaStream_t stream;
+};
+
+// Simple non-owning stream wrapper that allows implicit conversion to cudaStream_t and cuda::stream_ref.
+struct stream_wrapper_ref
+{
+  stream_wrapper_ref(cudaStream_t s)
+      : stream(s)
+  {}
+
+  operator cudaStream_t() const
+  {
+    return stream;
+  }
+  operator cuda::stream_ref() const
+  {
+    return cuda::stream_ref(stream);
+  }
+
+  cudaStream_t stream;
+};
+
+template <typename Wrapper, typename ExecutionPolicy>
+void TestOnStream(ExecutionPolicy policy)
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector v(3);
+  v[0] = 1;
+  v[1] = -2;
+  v[2] = 3;
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  Wrapper wrapper(s);
+
+  auto streampolicy = policy.on(wrapper);
+
+  ASSERT_EQUAL(thrust::reduce(streampolicy, v.begin(), v.end()), 2);
+
+  cudaStreamDestroy(s);
+}
+
+void TestCudartStreamSync()
+{
+  TestOnStream<stream_wrapper>(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestCudartStreamSync);
+
+void TestCudartStreamNoSync()
+{
+  TestOnStream<stream_wrapper>(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestCudartStreamNoSync);
+
+void TestCudaStreamRefSync()
+{
+  TestOnStream<stream_wrapper_ref>(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestCudaStreamRefSync);
+
+void TestCudaStreamRefNoSync()
+{
+  TestOnStream<stream_wrapper_ref>(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestCudaStreamRefNoSync);

cccl_upstream/thrust/testing/cuda/swap_ranges.cu

@@ -0,0 +1,64 @@
+#include <thrust/execution_policy.h>
+#include <thrust/swap.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
+__global__ void swap_ranges_kernel(ExecutionPolicy exec, Iterator1 first1, Iterator1 last1, Iterator2 first2)
+{
+  thrust::swap_ranges(exec, first1, last1, first2);
+}
+
+template <typename ExecutionPolicy>
+void TestSwapRangesDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector v1{0, 1, 2, 3, 4};
+  Vector v2{5, 6, 7, 8, 9};
+  Vector v1_ref(v2);
+  Vector v2_ref(v1);
+
+  swap_ranges_kernel<<<1, 1>>>(exec, v1.begin(), v1.end(), v2.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(v1, v1_ref);
+  ASSERT_EQUAL(v2, v2_ref);
+}
+
+void TestSwapRangesDeviceSeq()
+{
+  TestSwapRangesDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestSwapRangesDeviceSeq);
+
+void TestSwapRangesDeviceDevice()
+{
+  TestSwapRangesDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestSwapRangesDeviceDevice);
+#endif
+
+void TestSwapRangesCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector v1{0, 1, 2, 3, 4};
+  Vector v2{5, 6, 7, 8, 9};
+  Vector v1_ref(v2);
+  Vector v2_ref(v1);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::swap_ranges(thrust::cuda::par.on(s), v1.begin(), v1.end(), v2.begin());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(v1, v1_ref);
+  ASSERT_EQUAL(v2, v2_ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestSwapRangesCudaStreams);

cccl_upstream/thrust/testing/cuda/tabulate.cu

@@ -0,0 +1,95 @@
+#include <thrust/execution_policy.h>
+#include <thrust/functional.h>
+#include <thrust/tabulate.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename Function>
+__global__ void tabulate_kernel(ExecutionPolicy exec, Iterator first, Iterator last, Function f)
+{
+  thrust::tabulate(exec, first, last, f);
+}
+
+template <typename ExecutionPolicy>
+void TestTabulateDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using namespace thrust::placeholders;
+  using T = typename Vector::value_type;
+
+  Vector v(5);
+
+  tabulate_kernel<<<1, 1>>>(exec, v.begin(), v.end(), ::cuda::std::identity{});
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  Vector ref{0, 1, 2, 3, 4};
+  ASSERT_EQUAL(v, ref);
+
+  tabulate_kernel<<<1, 1>>>(exec, v.begin(), v.end(), -_1);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ref = {0, -1, -2, -3, -4};
+  ASSERT_EQUAL(v, ref);
+
+  tabulate_kernel<<<1, 1>>>(exec, v.begin(), v.end(), _1 * _1 * _1);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ref = {0, 1, 8, 27, 64};
+  ASSERT_EQUAL(v, ref);
+}
+
+void TestTabulateDeviceSeq()
+{
+  TestTabulateDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestTabulateDeviceSeq);
+
+void TestTabulateDeviceDevice()
+{
+  TestTabulateDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestTabulateDeviceDevice);
+#endif
+
+void TestTabulateCudaStreams()
+{
+  using namespace thrust::placeholders;
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector v(5);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::tabulate(thrust::cuda::par.on(s), v.begin(), v.end(), ::cuda::std::identity{});
+  cudaStreamSynchronize(s);
+
+  Vector ref{0, 1, 2, 3, 4};
+  ASSERT_EQUAL(v, ref);
+
+  thrust::tabulate(thrust::cuda::par.on(s), v.begin(), v.end(), -_1);
+  cudaStreamSynchronize(s);
+
+  ref = {0, -1, -2, -3, -4};
+  ASSERT_EQUAL(v, ref);
+
+  thrust::tabulate(thrust::cuda::par.on(s), v.begin(), v.end(), _1 * _1 * _1);
+  cudaStreamSynchronize(s);
+
+  ref = {0, 1, 8, 27, 64};
+  ASSERT_EQUAL(v, ref);
+
+  cudaStreamSynchronize(s);
+}
+DECLARE_UNITTEST(TestTabulateCudaStreams);

cccl_upstream/thrust/testing/cuda/transform.cu

@@ -0,0 +1,499 @@
+#include <thrust/execution_policy.h>
+#include <thrust/transform.h>
+
+#include <cuda/iterator>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Function, typename Iterator3>
+__global__ void transform_kernel(
+  ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result1, Function f, Iterator3 result2)
+{
+  *result2 = thrust::transform(exec, first, last, result1, f);
+}
+
+template <typename ExecutionPolicy>
+void TestTransformUnaryDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = typename Vector::value_type;
+
+  typename Vector::iterator iter;
+
+  Vector input{1, -2, 3};
+  Vector output(3);
+  Vector result{-1, 2, -3};
+
+  thrust::device_vector<typename Vector::iterator> iter_vec(1);
+
+  transform_kernel<<<1, 1>>>(
+    exec, input.begin(), input.end(), output.begin(), ::cuda::std::negate<T>(), iter_vec.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  iter = iter_vec[0];
+
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(output, result);
+}
+
+void TestTransformUnaryDeviceSeq()
+{
+  TestTransformUnaryDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestTransformUnaryDeviceSeq);
+
+void TestTransformUnaryDeviceDevice()
+{
+  TestTransformUnaryDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestTransformUnaryDeviceDevice);
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Function,
+          typename Predicate,
+          typename Iterator3>
+__global__ void transform_if_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first,
+  Iterator1 last,
+  Iterator2 result1,
+  Function f,
+  Predicate pred,
+  Iterator3 result2)
+{
+  *result2 = thrust::transform_if(exec, first, last, result1, f, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestTransformIfUnaryNoStencilDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = typename Vector::value_type;
+
+  typename Vector::iterator iter;
+
+  Vector input{0, -2, 0};
+  Vector output{-1, -2, -3};
+  Vector result{-1, 2, -3};
+
+  thrust::device_vector<typename Vector::iterator> iter_vec(1);
+
+  transform_if_kernel<<<1, 1>>>(
+    exec,
+    input.begin(),
+    input.end(),
+    output.begin(),
+    ::cuda::std::negate<T>(),
+    ::cuda::std::identity{},
+    iter_vec.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  iter = iter_vec[0];
+
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(output, result);
+}
+
+void TestTransformIfUnaryNoStencilDeviceSeq()
+{
+  TestTransformIfUnaryNoStencilDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestTransformIfUnaryNoStencilDeviceSeq);
+
+void TestTransformIfUnaryNoStencilDeviceDevice()
+{
+  TestTransformIfUnaryNoStencilDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestTransformIfUnaryNoStencilDeviceDevice);
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Function,
+          typename Predicate,
+          typename Iterator4>
+__global__ void transform_if_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first,
+  Iterator1 last,
+  Iterator2 stencil_first,
+  Iterator3 result1,
+  Function f,
+  Predicate pred,
+  Iterator4 result2)
+{
+  *result2 = thrust::transform_if(exec, first, last, stencil_first, result1, f, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestTransformIfUnaryDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = typename Vector::value_type;
+
+  typename Vector::iterator iter;
+
+  Vector input{1, -2, 3};
+  Vector stencil{1, 0, 1};
+  Vector output{1, 2, 3};
+  Vector result{-1, 2, -3};
+
+  thrust::device_vector<typename Vector::iterator> iter_vec(1);
+
+  transform_if_kernel<<<1, 1>>>(
+    exec,
+    input.begin(),
+    input.end(),
+    stencil.begin(),
+    output.begin(),
+    ::cuda::std::negate<T>(),
+    ::cuda::std::identity{},
+    iter_vec.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  iter = iter_vec[0];
+
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(output, result);
+}
+
+void TestTransformIfUnaryDeviceSeq()
+{
+  TestTransformIfUnaryDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestTransformIfUnaryDeviceSeq);
+
+void TestTransformIfUnaryDeviceDevice()
+{
+  TestTransformIfUnaryDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestTransformIfUnaryDeviceDevice);
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Function,
+          typename Iterator4>
+__global__ void transform_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first1,
+  Iterator1 last1,
+  Iterator2 first2,
+  Iterator3 result1,
+  Function f,
+  Iterator4 result2)
+{
+  *result2 = thrust::transform(exec, first1, last1, first2, result1, f);
+}
+
+template <typename ExecutionPolicy>
+void TestTransformBinaryDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = typename Vector::value_type;
+
+  typename Vector::iterator iter;
+
+  Vector input1{1, -2, 3};
+  Vector input2{-4, 5, 6};
+  Vector output(3);
+  Vector result{5, -7, -3};
+
+  thrust::device_vector<typename Vector::iterator> iter_vec(1);
+
+  transform_kernel<<<1, 1>>>(
+    exec, input1.begin(), input1.end(), input2.begin(), output.begin(), ::cuda::std::minus<T>(), iter_vec.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  iter = iter_vec[0];
+
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input1.size());
+  ASSERT_EQUAL(output, result);
+}
+
+void TestTransformBinaryDeviceSeq()
+{
+  TestTransformBinaryDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestTransformBinaryDeviceSeq);
+
+void TestTransformBinaryDeviceDevice()
+{
+  TestTransformBinaryDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestTransformBinaryDeviceDevice);
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Iterator4,
+          typename Function,
+          typename Predicate,
+          typename Iterator5>
+__global__ void transform_if_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first1,
+  Iterator1 last1,
+  Iterator2 first2,
+  Iterator3 stencil_first,
+  Iterator4 result1,
+  Function f,
+  Predicate pred,
+  Iterator5 result2)
+{
+  *result2 = thrust::transform_if(exec, first1, last1, first2, stencil_first, result1, f, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestTransformIfBinaryDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = typename Vector::value_type;
+
+  typename Vector::iterator iter;
+
+  Vector input1{1, -2, 3};
+  Vector input2{-4, 5, 6};
+  Vector stencil{0, 1, 0};
+  Vector output{1, 2, 3};
+  Vector result{5, 2, -3};
+
+  ::cuda::std::identity identity;
+
+  thrust::device_vector<typename Vector::iterator> iter_vec(1);
+
+  transform_if_kernel<<<1, 1>>>(
+    exec,
+    input1.begin(),
+    input1.end(),
+    input2.begin(),
+    stencil.begin(),
+    output.begin(),
+    ::cuda::std::minus<T>(),
+    ::cuda::std::not_fn(identity),
+    iter_vec.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  iter = iter_vec[0];
+
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input1.size());
+  ASSERT_EQUAL(output, result);
+}
+
+void TestTransformIfBinaryDeviceSeq()
+{
+  TestTransformIfBinaryDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestTransformIfBinaryDeviceSeq);
+
+void TestTransformIfBinaryDeviceDevice()
+{
+  TestTransformIfBinaryDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestTransformIfBinaryDeviceDevice);
+#endif
+
+void TestTransformUnaryCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector::iterator iter;
+
+  Vector input{1, -2, 3};
+  Vector output(3);
+  Vector result{-1, 2, -3};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  iter =
+    thrust::transform(thrust::cuda::par.on(s), input.begin(), input.end(), output.begin(), ::cuda::std::negate<T>());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(output, result);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestTransformUnaryCudaStreams);
+
+void TestTransformBinaryCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector::iterator iter;
+
+  Vector input1{1, -2, 3};
+  Vector input2{-4, 5, 6};
+  Vector output(3);
+  Vector result{5, -7, -3};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  iter = thrust::transform(
+    thrust::cuda::par.on(s), input1.begin(), input1.end(), input2.begin(), output.begin(), ::cuda::std::minus<T>());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input1.size());
+  ASSERT_EQUAL(output, result);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestTransformBinaryCudaStreams);
+
+struct sum_five
+{
+  _CCCL_HOST_DEVICE auto operator()(std::int8_t a, std::int16_t b, std::int32_t c, std::int64_t d, float e) const
+    -> double
+  {
+    return static_cast<double>(a) + static_cast<double>(b) + static_cast<double>(c) + static_cast<double>(d)
+         + static_cast<double>(e);
+  }
+};
+
+// we specialize zip_function for sum_five, but do nothing in the call operator so the test below would fail if the
+// zip_function is actually called (and not unwrapped)
+THRUST_NAMESPACE_BEGIN
+template <>
+class zip_function<sum_five>
+{
+public:
+  _CCCL_HOST_DEVICE zip_function(sum_five func)
+      : func(func)
+  {}
+
+  _CCCL_HOST_DEVICE sum_five& underlying_function() const
+  {
+    return func;
+  }
+
+  template <typename Tuple>
+  _CCCL_HOST_DEVICE double operator()(Tuple&& t) const
+  {
+    // not calling func, so we would get a wrong result if we were called
+    return 0;
+  }
+
+private:
+  mutable sum_five func;
+};
+THRUST_NAMESPACE_END
+
+// test that the cuda_cub backend of Thrust unwraps zip_iterators/zip_functions into their input streams
+void TestTransformThrustZipIteratorUnwrapping()
+{
+  constexpr int num_items = 100;
+  thrust::device_vector<std::int8_t> a(num_items, 1);
+  thrust::device_vector<std::int16_t> b(num_items, 2);
+  thrust::device_vector<std::int32_t> c(num_items, 3);
+  thrust::device_vector<std::int64_t> d(num_items, 4);
+  thrust::device_vector<float> e(num_items, 5);
+
+  thrust::device_vector<double> result(num_items);
+  // SECTION("once") // TODO(bgruber): enable sections when we migrate to Catch2
+  {
+    const auto z = thrust::make_zip_iterator(a.begin(), b.begin(), c.begin(), d.begin(), e.begin());
+    thrust::transform(z, z + num_items, result.begin(), thrust::make_zip_function(sum_five{}));
+
+    // compute reference and verify
+    thrust::device_vector<double> reference(num_items, 1 + 2 + 3 + 4 + 5);
+    ASSERT_EQUAL(reference, result);
+  }
+  // SECTION("trice")
+  {
+    const auto z = thrust::make_zip_iterator(
+      thrust::make_zip_iterator(thrust::make_zip_iterator(a.begin(), b.begin(), c.begin(), d.begin(), e.begin())));
+    thrust::transform(z,
+                      z + num_items,
+                      result.begin(),
+                      thrust::make_zip_function(thrust::make_zip_function(thrust::make_zip_function(sum_five{}))));
+
+    // compute reference and verify
+    thrust::device_vector<double> reference(num_items, 1 + 2 + 3 + 4 + 5);
+    ASSERT_EQUAL(reference, result);
+  }
+}
+DECLARE_UNITTEST(TestTransformThrustZipIteratorUnwrapping);
+
+// we specialize zip_function for sum_five, but do nothing in the call operator so the test below would fail if the
+// zip_function is actually called (and not unwrapped)
+_CCCL_BEGIN_NAMESPACE_CUDA
+template <>
+class zip_function<sum_five>
+{
+private:
+  sum_five __fun_;
+
+public:
+  zip_function() = default;
+
+  _CCCL_HOST_DEVICE zip_function(sum_five&& func) noexcept
+      : __fun_(::cuda::std::move(func))
+  {}
+
+  template <typename _Tuple>
+  _CCCL_HOST_DEVICE decltype(auto) operator()(_Tuple&& __tuple) const noexcept
+  {
+    // not calling func, just return a default ctored element, so we would get a wrong result if we were called
+    return decltype(::cuda::std::apply(__fun_, ::cuda::std::forward<_Tuple>(__tuple))){};
+  }
+
+  _CCCL_HOST_DEVICE sum_five& __fun() noexcept
+  {
+    return __fun_;
+  }
+
+  _CCCL_HOST_DEVICE const sum_five& __fun() const noexcept
+  {
+    return __fun_;
+  }
+};
+_CCCL_END_NAMESPACE_CUDA
+
+// test that the cuda_cub backend of Thrust unwraps zip_iterators/zip_functions into their input streams
+void TestTransformCudaZipIteratorUnwrapping()
+{
+  constexpr int num_items = 100;
+  thrust::device_vector<std::int8_t> a(num_items, 1);
+  thrust::device_vector<std::int16_t> b(num_items, 2);
+  thrust::device_vector<std::int32_t> c(num_items, 3);
+  thrust::device_vector<std::int64_t> d(num_items, 4);
+  thrust::device_vector<float> e(num_items, 5);
+
+  thrust::device_vector<double> result(num_items);
+  // SECTION("once") // TODO(bgruber): enable sections when we migrate to Catch2
+  {
+    const auto z = cuda::make_zip_iterator(a.begin(), b.begin(), c.begin(), d.begin(), e.begin());
+    thrust::transform(z, z + num_items, result.begin(), cuda::zip_function(sum_five{}));
+
+    // compute reference and verify
+    thrust::device_vector<double> reference(num_items, 1 + 2 + 3 + 4 + 5);
+    ASSERT_EQUAL(reference, result);
+  }
+  // SECTION("trice")
+  {
+    const auto z = cuda::make_zip_iterator(
+      cuda::make_zip_iterator(cuda::make_zip_iterator(a.begin(), b.begin(), c.begin(), d.begin(), e.begin())));
+    thrust::transform(
+      z, z + num_items, result.begin(), cuda::zip_function<cuda::zip_function<cuda::zip_function<sum_five>>>{});
+
+    // compute reference and verify
+    thrust::device_vector<double> reference(num_items, 1 + 2 + 3 + 4 + 5);
+    ASSERT_EQUAL(reference, result);
+  }
+}
+DECLARE_UNITTEST(TestTransformCudaZipIteratorUnwrapping);

cccl_upstream/thrust/testing/cuda/transform_iterator.cmake

@@ -0,0 +1,22 @@
+# There are extra ; in the extended lambda implementation
+target_compile_options(
+  ${test_target}
+  PRIVATE $<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:--extended-lambda>
+)
+
+# this check is actually not correct, because we must check the host compiler, not the CXX compiler.
+# We rely on that those are usually the same ;)
+if (
+  "Clang" STREQUAL "${CMAKE_CXX_COMPILER_ID}"
+  AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13
+)
+  # When clang >= 13 is used as host compiler, we get the following warning:
+  #   nvcc_internal_extended_lambda_implementation:312:22: error: definition of implicit copy constructor for '__nv_hdl_wrapper_t<false, true, false, __nv_dl_tag<void (*)(), &TestAddressStabilityLambda, 2>, int (const int &)>' is deprecated because it has a user-declared copy assignment operator [-Werror,-Wdeprecated-copy]
+  #   312 | __nv_hdl_wrapper_t & operator=(const __nv_hdl_wrapper_t &in) = delete;
+  #   |                      ^
+  # Let's suppress it until NVBug 4980157 is resolved.
+  target_compile_options(
+    ${test_target}
+    PRIVATE $<$<COMPILE_LANG_AND_ID:CUDA,NVIDIA>:-Wno-deprecated-copy>
+  )
+endif()

cccl_upstream/thrust/testing/cuda/transform_iterator.cu

@@ -0,0 +1,19 @@
+#include <thrust/device_vector.h>
+#include <thrust/host_vector.h>
+#include <thrust/iterator/transform_iterator.h>
+#include <thrust/logical.h>
+
+#include <unittest/unittest.h>
+
+// see also: https://github.com/NVIDIA/cccl/issues/3541
+void TestTransformWithLambda()
+{
+  auto l = [] __host__ __device__(int v) { return v < 4; };
+  thrust::host_vector<int> A{1, 2, 3, 4, 5, 6, 7};
+  ASSERT_EQUAL(thrust::any_of(A.begin(), A.end(), l), true);
+
+  thrust::device_vector<int> B{1, 2, 3, 4, 5, 6, 7};
+  ASSERT_EQUAL(thrust::any_of(B.begin(), B.end(), l), true);
+}
+
+DECLARE_UNITTEST(TestTransformWithLambda);

cccl_upstream/thrust/testing/cuda/transform_reduce.cu

@@ -0,0 +1,65 @@
+#include <thrust/execution_policy.h>
+#include <thrust/transform_reduce.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Function1, typename T, typename Function2, typename Iterator2>
+__global__ void transform_reduce_kernel(
+  ExecutionPolicy exec, Iterator1 first, Iterator1 last, Function1 f1, T init, Function2 f2, Iterator2 result)
+{
+  *result = thrust::transform_reduce(exec, first, last, f1, init, f2);
+}
+
+template <typename ExecutionPolicy>
+void TestTransformReduceDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = typename Vector::value_type;
+
+  Vector data{1, -2, 3};
+  T init = 10;
+
+  thrust::device_vector<T> result(1);
+
+  transform_reduce_kernel<<<1, 1>>>(
+    exec, data.begin(), data.end(), ::cuda::std::negate<T>(), init, ::cuda::std::plus<T>(), result.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(8, (T) result[0]);
+}
+
+void TestTransformReduceDeviceSeq()
+{
+  TestTransformReduceDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestTransformReduceDeviceSeq);
+
+void TestTransformReduceDeviceDevice()
+{
+  TestTransformReduceDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestTransformReduceDeviceDevice);
+#endif
+
+void TestTransformReduceCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{1, -2, 3};
+  T init = 10;
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  T result = thrust::transform_reduce(
+    thrust::cuda::par.on(s), data.begin(), data.end(), ::cuda::std::negate<T>(), init, ::cuda::std::plus<T>());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(8, result);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestTransformReduceCudaStreams);

cccl_upstream/thrust/testing/cuda/transform_scan.cu

@@ -0,0 +1,378 @@
+#include <thrust/execution_policy.h>
+#include <thrust/transform_scan.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Function1,
+          typename Function2,
+          typename Iterator3>
+__global__ void transform_inclusive_scan_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first,
+  Iterator1 last,
+  Iterator2 result1,
+  Function1 f1,
+  Function2 f2,
+  Iterator3 result2)
+{
+  *result2 = thrust::transform_inclusive_scan(exec, first, last, result1, f1, f2);
+}
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Function1,
+          typename T,
+          typename Function2,
+          typename Iterator3>
+__global__ void transform_inclusive_scan_init_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first,
+  Iterator1 last,
+  Iterator2 result1,
+  Function1 f1,
+  T init,
+  Function2 f2,
+  Iterator3 result2)
+{
+  *result2 = thrust::transform_inclusive_scan(exec, first, last, result1, f1, init, f2);
+}
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Function1,
+          typename T,
+          typename Function2,
+          typename Iterator3>
+__global__ void transform_exclusive_scan_kernel(
+  ExecutionPolicy exec,
+  Iterator1 first,
+  Iterator1 last,
+  Iterator2 result,
+  Function1 f1,
+  T init,
+  Function2 f2,
+  Iterator3 result2)
+{
+  *result2 = thrust::transform_exclusive_scan(exec, first, last, result, f1, init, f2);
+}
+
+template <typename ExecutionPolicy>
+void TestTransformScanDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = typename Vector::value_type;
+
+  typename Vector::iterator iter;
+
+  Vector input{1, 3, -2, 4, -5};
+  Vector ref{-1, -4, -2, -6, -1};
+  Vector output(5);
+
+  Vector input_copy(input);
+
+  thrust::device_vector<typename Vector::iterator> iter_vec(1);
+
+  // inclusive scan
+  transform_inclusive_scan_kernel<<<1, 1>>>(
+    exec,
+    input.begin(),
+    input.end(),
+    output.begin(),
+    ::cuda::std::negate<T>(),
+    ::cuda::std::plus<T>(),
+    iter_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  iter = iter_vec[0];
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(ref, output);
+
+  // inclusive scan with nonzero init
+  transform_inclusive_scan_init_kernel<<<1, 1>>>(
+    exec,
+    input.begin(),
+    input.end(),
+    output.begin(),
+    ::cuda::std::negate<T>(),
+    3,
+    ::cuda::std::plus<T>(),
+    iter_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  iter = iter_vec[0];
+  ref  = {2, -1, 1, -3, 2};
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(ref, output);
+
+  // exclusive scan with 0 init
+  transform_exclusive_scan_kernel<<<1, 1>>>(
+    exec,
+    input.begin(),
+    input.end(),
+    output.begin(),
+    ::cuda::std::negate<T>(),
+    0,
+    ::cuda::std::plus<T>(),
+    iter_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ref = {0, -1, -4, -2, -6};
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(ref, output);
+
+  // exclusive scan with nonzero init
+  transform_exclusive_scan_kernel<<<1, 1>>>(
+    exec,
+    input.begin(),
+    input.end(),
+    output.begin(),
+    ::cuda::std::negate<T>(),
+    3,
+    ::cuda::std::plus<T>(),
+    iter_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  iter = iter_vec[0];
+  ref  = {3, 2, -1, 1, -3};
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(ref, output);
+
+  // inplace inclusive scan
+  input = input_copy;
+  transform_inclusive_scan_kernel<<<1, 1>>>(
+    exec, input.begin(), input.end(), input.begin(), ::cuda::std::negate<T>(), ::cuda::std::plus<T>(), iter_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  iter = iter_vec[0];
+  ref  = {-1, -4, -2, -6, -1};
+  ASSERT_EQUAL(std::size_t(iter - input.begin()), input.size());
+  ASSERT_EQUAL(ref, input);
+
+  // inplace inclusive scan with init
+  input = input_copy;
+  transform_inclusive_scan_init_kernel<<<1, 1>>>(
+    exec,
+    input.begin(),
+    input.end(),
+    input.begin(),
+    ::cuda::std::negate<T>(),
+    3,
+    ::cuda::std::plus<T>(),
+    iter_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  iter = iter_vec[0];
+  ref  = {2, -1, 1, -3, 2};
+  ASSERT_EQUAL(std::size_t(iter - input.begin()), input.size());
+  ASSERT_EQUAL(ref, input);
+
+  // inplace exclusive scan with init
+  input = input_copy;
+  transform_exclusive_scan_kernel<<<1, 1>>>(
+    exec,
+    input.begin(),
+    input.end(),
+    input.begin(),
+    ::cuda::std::negate<T>(),
+    3,
+    ::cuda::std::plus<T>(),
+    iter_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  iter = iter_vec[0];
+  ref  = {3, 2, -1, 1, -3};
+  ASSERT_EQUAL(std::size_t(iter - input.begin()), input.size());
+  ASSERT_EQUAL(ref, input);
+}
+
+void TestTransformScanDeviceSeq()
+{
+  TestTransformScanDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestTransformScanDeviceSeq);
+
+void TestTransformScanDeviceDevice()
+{
+  TestTransformScanDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestTransformScanDeviceDevice);
+#endif
+
+void TestTransformScanCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector::iterator iter;
+
+  Vector input{1, 3, -2, 4, -5};
+  Vector result{-1, -4, -2, -6, -1};
+  Vector output(5);
+
+  Vector input_copy(input);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  // inclusive scan
+  iter = thrust::transform_inclusive_scan(
+    thrust::cuda::par.on(s),
+    input.begin(),
+    input.end(),
+    output.begin(),
+    ::cuda::std::negate<T>(),
+    ::cuda::std::plus<T>());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(output, result);
+
+  // inclusive scan with nonzero init
+  iter = thrust::transform_inclusive_scan(
+    thrust::cuda::par.on(s),
+    input.begin(),
+    input.end(),
+    output.begin(),
+    ::cuda::std::negate<T>(),
+    3,
+    ::cuda::std::plus<T>());
+  cudaStreamSynchronize(s);
+
+  result = {2, -1, 1, -3, 2};
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(output, result);
+
+  // exclusive scan with 0 init
+  iter = thrust::transform_exclusive_scan(
+    thrust::cuda::par.on(s),
+    input.begin(),
+    input.end(),
+    output.begin(),
+    ::cuda::std::negate<T>(),
+    0,
+    ::cuda::std::plus<T>());
+  cudaStreamSynchronize(s);
+
+  result = {0, -1, -4, -2, -6};
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(output, result);
+
+  // exclusive scan with nonzero init
+  iter = thrust::transform_exclusive_scan(
+    thrust::cuda::par.on(s),
+    input.begin(),
+    input.end(),
+    output.begin(),
+    ::cuda::std::negate<T>(),
+    3,
+    ::cuda::std::plus<T>());
+  cudaStreamSynchronize(s);
+
+  result = {3, 2, -1, 1, -3};
+  ASSERT_EQUAL(std::size_t(iter - output.begin()), input.size());
+  ASSERT_EQUAL(input, input_copy);
+  ASSERT_EQUAL(output, result);
+
+  // inplace inclusive scan
+  input = input_copy;
+  iter  = thrust::transform_inclusive_scan(
+    thrust::cuda::par.on(s),
+    input.begin(),
+    input.end(),
+    input.begin(),
+    ::cuda::std::negate<T>(),
+    ::cuda::std::plus<T>());
+  cudaStreamSynchronize(s);
+
+  result = {-1, -4, -2, -6, -1};
+  ASSERT_EQUAL(std::size_t(iter - input.begin()), input.size());
+  ASSERT_EQUAL(input, result);
+
+  // inplace inclusive scan with init
+  input = input_copy;
+  iter  = thrust::transform_inclusive_scan(
+    thrust::cuda::par.on(s),
+    input.begin(),
+    input.end(),
+    input.begin(),
+    ::cuda::std::negate<T>(),
+    3,
+    ::cuda::std::plus<T>());
+  cudaStreamSynchronize(s);
+
+  result = {2, -1, 1, -3, 2};
+  ASSERT_EQUAL(std::size_t(iter - input.begin()), input.size());
+  ASSERT_EQUAL(input, result);
+
+  // inplace exclusive scan with init
+  input = input_copy;
+  iter  = thrust::transform_exclusive_scan(
+    thrust::cuda::par.on(s),
+    input.begin(),
+    input.end(),
+    input.begin(),
+    ::cuda::std::negate<T>(),
+    3,
+    ::cuda::std::plus<T>());
+  cudaStreamSynchronize(s);
+
+  result = {3, 2, -1, 1, -3};
+  ASSERT_EQUAL(std::size_t(iter - input.begin()), input.size());
+  ASSERT_EQUAL(input, result);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestTransformScanCudaStreams);
+
+void TestTransformScanConstAccumulator()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector::iterator iter;
+
+  Vector input{1, 3, -2, 4, -5};
+  Vector reference(5);
+  Vector output(5);
+
+  thrust::transform_inclusive_scan(
+    input.begin(), input.end(), output.begin(), ::cuda::std::identity{}, ::cuda::std::plus<T>());
+  thrust::inclusive_scan(input.begin(), input.end(), reference.begin(), ::cuda::std::plus<T>());
+
+  ASSERT_EQUAL(output, reference);
+}
+DECLARE_UNITTEST(TestTransformScanConstAccumulator);

cccl_upstream/thrust/testing/cuda/trivial_copy_memcpy_default.cu

@@ -0,0 +1,43 @@
+#include <thrust/system/cuda/detail/util.h>
+
+#include <unittest/unittest.h>
+
+// These tests verify that trivial_copy_from_device and trivial_copy_to_device
+// use cudaMemcpyDefault instead of cudaMemcpyDeviceToHost/cudaMemcpyHostToDevice.
+//
+// With explicit copy directions, cudaMemcpyAsync fails when the actual memory
+// location doesn't match the expected direction (e.g., cudaMemcpyDeviceToHost
+// with a host source pointer returns cudaErrorInvalidValue).
+//
+// With cudaMemcpyDefault, CUDA determines the correct direction at runtime,
+// which is necessary when device-accessible pointers may point to host memory.
+
+void TestTrivialCopyFromDevice_HostSource()
+{
+  int src[]  = {0, 10, 20, 30, 40};
+  int dst[5] = {};
+
+  cudaError_t status = thrust::cuda_cub::trivial_copy_from_device(dst, src, 5, cudaStreamDefault);
+
+  ASSERT_EQUAL(status, cudaSuccess);
+  for (int i = 0; i < 5; i++)
+  {
+    ASSERT_EQUAL(dst[i], i * 10);
+  }
+}
+DECLARE_UNITTEST(TestTrivialCopyFromDevice_HostSource);
+
+void TestTrivialCopyToDevice_HostDest()
+{
+  int src[]  = {0, 100, 200, 300, 400};
+  int dst[5] = {};
+
+  cudaError_t status = thrust::cuda_cub::trivial_copy_to_device(dst, src, 5, cudaStreamDefault);
+
+  ASSERT_EQUAL(status, cudaSuccess);
+  for (int i = 0; i < 5; i++)
+  {
+    ASSERT_EQUAL(dst[i], i * 100);
+  }
+}
+DECLARE_UNITTEST(TestTrivialCopyToDevice_HostDest);

cccl_upstream/thrust/testing/cuda/uninitialized_copy.cu

@@ -0,0 +1,117 @@
+#include <thrust/execution_policy.h>
+#include <thrust/uninitialized_copy.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
+__global__ void uninitialized_copy_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result)
+{
+  thrust::uninitialized_copy(exec, first, last, result);
+}
+
+template <typename ExecutionPolicy>
+void TestUninitializedCopyDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector v1{0, 1, 2, 3, 4};
+
+  // copy to Vector
+  Vector v2(5);
+  uninitialized_copy_kernel<<<1, 1>>>(exec, v1.begin(), v1.end(), v2.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  Vector ref{0, 1, 2, 3, 4};
+  ASSERT_EQUAL(v2, ref);
+}
+
+void TestUninitializedCopyDeviceSeq()
+{
+  TestUninitializedCopyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestUninitializedCopyDeviceSeq);
+
+void TestUninitializedCopyDeviceDevice()
+{
+  TestUninitializedCopyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestUninitializedCopyDeviceDevice);
+#endif
+
+void TestUninitializedCopyCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector v1{0, 1, 2, 3, 4};
+
+  // copy to Vector
+  Vector v2(5);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::uninitialized_copy(thrust::cuda::par.on(s), v1.begin(), v1.end(), v2.begin());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(v2, v1);
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestUninitializedCopyCudaStreams);
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Size, typename Iterator2>
+__global__ void uninitialized_copy_n_kernel(ExecutionPolicy exec, Iterator1 first, Size n, Iterator2 result)
+{
+  thrust::uninitialized_copy_n(exec, first, n, result);
+}
+
+template <typename ExecutionPolicy>
+void TestUninitializedCopyNDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector v1{0, 1, 2, 3, 4};
+
+  // copy to Vector
+  Vector v2(5);
+  uninitialized_copy_n_kernel<<<1, 1>>>(exec, v1.begin(), v1.size(), v2.begin());
+  cudaError_t const err = cudaDeviceSynchronize();
+  ASSERT_EQUAL(cudaSuccess, err);
+
+  ASSERT_EQUAL(v2, v1);
+}
+
+void TestUninitializedCopyNDeviceSeq()
+{
+  TestUninitializedCopyNDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestUninitializedCopyNDeviceSeq);
+
+void TestUninitializedCopyNDeviceDevice()
+{
+  TestUninitializedCopyNDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestUninitializedCopyNDeviceDevice);
+#endif
+
+void TestUninitializedCopyNCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+
+  Vector v1{0, 1, 2, 3, 4};
+
+  // copy to Vector
+  Vector v2(5);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::uninitialized_copy_n(thrust::cuda::par.on(s), v1.begin(), v1.size(), v2.begin());
+  cudaStreamSynchronize(s);
+  ASSERT_EQUAL(v2, v1);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestUninitializedCopyNCudaStreams);

cccl_upstream/thrust/testing/cuda/uninitialized_fill.cu

@@ -0,0 +1,205 @@
+#include <thrust/execution_policy.h>
+#include <thrust/uninitialized_fill.h>
+
+#include <unittest/unittest.h>
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator, typename T>
+__global__ void uninitialized_fill_kernel(ExecutionPolicy exec, Iterator first, Iterator last, T val)
+{
+  thrust::uninitialized_fill(exec, first, last, val);
+}
+
+template <typename ExecutionPolicy>
+void TestUninitializedFillDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector v{0, 1, 2, 3, 4};
+  T sub(7);
+
+  uninitialized_fill_kernel<<<1, 1>>>(exec, v.begin() + 1, v.begin() + 4, sub);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  Vector ref{0, sub, sub, sub, 4};
+  ASSERT_EQUAL(v, ref);
+
+  sub = 8;
+
+  uninitialized_fill_kernel<<<1, 1>>>(exec, v.begin() + 0, v.begin() + 3, sub);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ref = {sub, sub, sub, 7, 4};
+  ASSERT_EQUAL(v, ref);
+
+  sub = 9;
+
+  uninitialized_fill_kernel<<<1, 1>>>(exec, v.begin() + 2, v.end(), sub);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ref = {8, 8, sub, sub, 9};
+  ASSERT_EQUAL(v, ref);
+
+  sub = 1;
+
+  uninitialized_fill_kernel<<<1, 1>>>(exec, v.begin(), v.end(), sub);
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ref = Vector(5, sub);
+  ASSERT_EQUAL(v, ref);
+}
+
+void TestUninitializedFillDeviceSeq()
+{
+  TestUninitializedFillDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestUninitializedFillDeviceSeq);
+
+void TestUninitializedFillDeviceDevice()
+{
+  TestUninitializedFillDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestUninitializedFillDeviceDevice);
+#endif
+
+void TestUninitializedFillCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector v{0, 1, 2, 3, 4};
+  T sub(7);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::uninitialized_fill(thrust::cuda::par.on(s), v.begin(), v.end(), sub);
+  cudaStreamSynchronize(s);
+
+  Vector ref(v.size(), sub);
+  ASSERT_EQUAL(v, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestUninitializedFillCudaStreams);
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Size, typename T, typename Iterator2>
+__global__ void uninitialized_fill_n_kernel(ExecutionPolicy exec, Iterator1 first, Size n, T val, Iterator2 result)
+{
+  *result = thrust::uninitialized_fill_n(exec, first, n, val);
+}
+
+template <typename ExecutionPolicy>
+void TestUninitializedFillNDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector v{0, 1, 2, 3, 4};
+  T sub(7);
+
+  thrust::device_vector<Vector::iterator> iter_vec(1);
+
+  uninitialized_fill_n_kernel<<<1, 1>>>(exec, v.begin() + 1, 3, sub, iter_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  Vector::iterator iter = iter_vec[0];
+
+  Vector ref{0, sub, sub, sub, 4};
+  ASSERT_EQUAL(v, ref);
+  ASSERT_EQUAL_QUIET(v.begin() + 4, iter);
+
+  sub = 8;
+
+  uninitialized_fill_n_kernel<<<1, 1>>>(exec, v.begin() + 0, 3, sub, iter_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  iter = iter_vec[0];
+
+  ref = {sub, sub, sub, 7, 4};
+  ASSERT_EQUAL(v, ref);
+  ASSERT_EQUAL_QUIET(v.begin() + 3, iter);
+
+  sub = 9;
+
+  uninitialized_fill_n_kernel<<<1, 1>>>(exec, v.begin() + 2, 3, sub, iter_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  iter = iter_vec[0];
+
+  ref = {8, 8, sub, sub, 9};
+  ASSERT_EQUAL(v, ref);
+  ASSERT_EQUAL_QUIET(v.end(), iter);
+
+  sub = 1;
+
+  uninitialized_fill_n_kernel<<<1, 1>>>(exec, v.begin(), v.size(), sub, iter_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  iter = iter_vec[0];
+
+  ref = Vector(5, sub);
+
+  ASSERT_EQUAL(v, ref);
+  ASSERT_EQUAL_QUIET(v.end(), iter);
+}
+
+void TestUninitializedFillNDeviceSeq()
+{
+  TestUninitializedFillNDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestUninitializedFillNDeviceSeq);
+
+void TestUninitializedFillNDeviceDevice()
+{
+  TestUninitializedFillNDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestUninitializedFillNDeviceDevice);
+#endif
+
+void TestUninitializedFillNCudaStreams()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector v{0, 1, 2, 3, 4};
+  T sub(7);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  thrust::uninitialized_fill_n(thrust::cuda::par.on(s), v.begin(), v.size(), sub);
+  cudaStreamSynchronize(s);
+
+  Vector ref(5, sub);
+  ASSERT_EQUAL(v, ref);
+
+  cudaStreamDestroy(s);
+}
+DECLARE_UNITTEST(TestUninitializedFillNCudaStreams);

cccl_upstream/thrust/testing/cuda/unique.cu

@@ -0,0 +1,478 @@
+#include <thrust/detail/raw_pointer_cast.h>
+#include <thrust/execution_policy.h>
+#include <thrust/iterator/counting_iterator.h>
+#include <thrust/unique.h>
+
+#include <unittest/unittest.h>
+
+template <typename T>
+struct div_n_equality_op
+{
+  T div;
+  __host__ __device__ bool operator()(const T x, const T& y) const
+  {
+    return (x / div) == (y / div);
+  }
+};
+
+template <typename T>
+struct multiply_n
+{
+  T multiplier;
+  __host__ __device__ T operator()(T x)
+  {
+    return x * multiplier;
+  }
+};
+
+struct check_valid_item_op
+{
+  ::cuda::std::uint32_t* error_counter{};
+  int expected_upper_bound{};
+
+  __device__ bool operator()(const int lhs, const int rhs) const
+  {
+    if (lhs > expected_upper_bound || rhs > expected_upper_bound)
+    {
+      if (error_counter)
+      {
+        atomicAdd(error_counter, 1);
+      }
+      return false;
+    }
+    return lhs == rhs;
+  }
+};
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
+__global__ void unique_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result)
+{
+  *result = thrust::unique(exec, first, last);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename BinaryPredicate, typename Iterator2>
+__global__ void
+unique_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, BinaryPredicate pred, Iterator2 result)
+{
+  *result = thrust::unique(exec, first, last, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestUniqueDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{11, 11, 12, 20, 29, 21, 21, 31, 31, 37};
+
+  thrust::device_vector<Vector::iterator> new_last_vec(1);
+  Vector::iterator new_last;
+
+  unique_kernel<<<1, 1>>>(exec, data.begin(), data.end(), new_last_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  new_last = new_last_vec[0];
+
+  ASSERT_EQUAL(new_last - data.begin(), 7);
+  data.erase(new_last, data.end());
+  Vector ref{11, 12, 20, 29, 21, 31, 37}; // should we consider calculating ref from std::algorithm if exists?
+  ASSERT_EQUAL(data, ref);
+
+  unique_kernel<<<1, 1>>>(exec, data.begin(), new_last, div_n_equality_op<T>{10}, new_last_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  new_last = new_last_vec[0];
+
+  ASSERT_EQUAL(new_last - data.begin(), 3);
+  data.erase(new_last, data.end());
+  ref = {11, 20, 31};
+  ASSERT_EQUAL(data, ref);
+}
+
+void TestUniqueDeviceSeq()
+{
+  TestUniqueDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestUniqueDeviceSeq);
+
+void TestUniqueDeviceDevice()
+{
+  TestUniqueDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestUniqueDeviceDevice);
+
+void TestUniqueDeviceNoSync()
+{
+  TestUniqueDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestUniqueDeviceNoSync);
+#endif
+
+template <typename ExecutionPolicy>
+void TestUniqueCudaStreams(ExecutionPolicy policy)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{11, 11, 12, 20, 29, 21, 21, 31, 31, 37};
+
+  thrust::device_vector<Vector::iterator> new_last_vec(1);
+  Vector::iterator new_last;
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  auto streampolicy = policy.on(s);
+
+  new_last = thrust::unique(streampolicy, data.begin(), data.end());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(new_last - data.begin(), 7);
+  data.erase(new_last, data.end());
+  Vector ref{11, 12, 20, 29, 21, 31, 37};
+  ASSERT_EQUAL(data, ref);
+
+  new_last = thrust::unique(streampolicy, data.begin(), new_last, div_n_equality_op<T>{10});
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(new_last - data.begin(), 3);
+  data.erase(new_last, data.end());
+  ref = {11, 20, 31};
+  ASSERT_EQUAL(data, ref);
+
+  cudaStreamDestroy(s);
+}
+
+void TestUniqueCudaStreamsSync()
+{
+  TestUniqueCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestUniqueCudaStreamsSync);
+
+void TestUniqueCudaStreamsNoSync()
+{
+  TestUniqueCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestUniqueCudaStreamsNoSync);
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Iterator3>
+__global__ void
+unique_copy_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result1, Iterator3 result2)
+{
+  *result2 = thrust::unique_copy(exec, first, last, result1);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename BinaryPredicate, typename Iterator3>
+__global__ void unique_copy_kernel(
+  ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result1, BinaryPredicate pred, Iterator3 result2)
+{
+  *result2 = thrust::unique_copy(exec, first, last, result1, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestUniqueCopyDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{11, 11, 12, 20, 29, 21, 21, 31, 31, 37};
+
+  Vector output(10, -1);
+
+  thrust::device_vector<Vector::iterator> new_last_vec(1);
+  Vector::iterator new_last;
+
+  unique_copy_kernel<<<1, 1>>>(exec, data.begin(), data.end(), output.begin(), new_last_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  new_last = new_last_vec[0];
+
+  ASSERT_EQUAL(new_last - output.begin(), 7);
+  output.erase(new_last, output.end());
+  Vector ref{11, 12, 20, 29, 21, 31, 37};
+  ASSERT_EQUAL(output, ref);
+
+  unique_copy_kernel<<<1, 1>>>(
+    exec, output.begin(), new_last, data.begin(), div_n_equality_op<T>{10}, new_last_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  new_last = new_last_vec[0];
+
+  ASSERT_EQUAL(new_last - data.begin(), 3);
+  data.erase(new_last, data.end());
+  ref = {11, 20, 31};
+  ASSERT_EQUAL(data, ref);
+}
+
+void TestUniqueCopyDeviceSeq()
+{
+  TestUniqueCopyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestUniqueCopyDeviceSeq);
+
+void TestUniqueCopyDeviceDevice()
+{
+  TestUniqueCopyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestUniqueCopyDeviceDevice);
+
+void TestUniqueCopyDeviceNoSync()
+{
+  TestUniqueCopyDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestUniqueCopyDeviceNoSync);
+#endif
+
+template <typename ExecutionPolicy>
+void TestUniqueCopyCudaStreams(ExecutionPolicy policy)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{11, 11, 12, 20, 29, 21, 21, 31, 31, 37};
+
+  Vector output(10, -1);
+
+  thrust::device_vector<Vector::iterator> new_last_vec(1);
+  Vector::iterator new_last;
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  auto streampolicy = policy.on(s);
+
+  new_last = thrust::unique_copy(streampolicy, data.begin(), data.end(), output.begin());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(new_last - output.begin(), 7);
+  output.erase(new_last, output.end());
+  Vector ref{11, 12, 20, 29, 21, 31, 37};
+  ASSERT_EQUAL(output, ref);
+
+  new_last = thrust::unique_copy(streampolicy, output.begin(), new_last, data.begin(), div_n_equality_op<T>{10});
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(new_last - data.begin(), 3);
+  data.erase(new_last, data.end());
+  ref = {11, 20, 31};
+  ASSERT_EQUAL(data, ref);
+
+  cudaStreamDestroy(s);
+}
+
+void TestUniqueCopyCudaStreamsSync()
+{
+  TestUniqueCopyCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestUniqueCopyCudaStreamsSync);
+
+void TestUniqueCopyCudaStreamsNoSync()
+{
+  TestUniqueCopyCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestUniqueCopyCudaStreamsNoSync);
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2>
+__global__ void unique_count_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, Iterator2 result)
+{
+  *result = thrust::unique_count(exec, first, last);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename BinaryPredicate, typename Iterator2>
+__global__ void
+unique_count_kernel(ExecutionPolicy exec, Iterator1 first, Iterator1 last, BinaryPredicate pred, Iterator2 result)
+{
+  *result = thrust::unique_count(exec, first, last, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestUniqueCountDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{11, 11, 12, 20, 29, 21, 21, 31, 31, 37};
+
+  Vector output(1, -1);
+
+  unique_count_kernel<<<1, 1>>>(exec, data.begin(), data.end(), output.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(output[0], 7);
+
+  unique_count_kernel<<<1, 1>>>(exec, data.begin(), data.end(), div_n_equality_op<T>{10}, output.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  ASSERT_EQUAL(output[0], 3);
+}
+
+void TestUniqueCountDeviceSeq()
+{
+  TestUniqueCountDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestUniqueCountDeviceSeq);
+
+void TestUniqueCountDeviceDevice()
+{
+  TestUniqueCountDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestUniqueCountDeviceDevice);
+
+void TestUniqueCountDeviceNoSync()
+{
+  TestUniqueCountDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestUniqueCountDeviceNoSync);
+#endif
+
+template <typename ExecutionPolicy>
+void TestUniqueCountCudaStreams(ExecutionPolicy policy)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector data{11, 11, 12, 20, 29, 21, 21, 31, 31, 37};
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  auto streampolicy = policy.on(s);
+
+  int result = thrust::unique_count(streampolicy, data.begin(), data.end());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(result, 7);
+
+  result = thrust::unique_count(streampolicy, data.begin(), data.end(), div_n_equality_op<T>{10});
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(result, 3);
+
+  cudaStreamDestroy(s);
+}
+
+void TestUniqueCountCudaStreamsSync()
+{
+  TestUniqueCountCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestUniqueCountCudaStreamsSync);
+
+void TestUniqueCountCudaStreamsNoSync()
+{
+  TestUniqueCountCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestUniqueCountCudaStreamsNoSync);
+
+void TestUniqueWithMagnitude(int magnitude)
+{
+  using offset_t      = std::int64_t;
+  using equality_op_t = div_n_equality_op<offset_t>;
+
+  offset_t run_length_of_equal_items = offset_t{10};
+  equality_op_t equality_op          = equality_op_t{run_length_of_equal_items};
+
+  // Prepare input
+  offset_t num_items = offset_t{1ull} << magnitude;
+  thrust::counting_iterator<offset_t> begin(offset_t{0});
+  auto end = begin + num_items;
+  ASSERT_EQUAL(static_cast<offset_t>(cuda::std::distance(begin, end)), num_items);
+
+  offset_t expected_num_unique = ::cuda::ceil_div(num_items, offset_t{10});
+  thrust::device_vector<offset_t> unique_out(expected_num_unique);
+  auto unique_out_end = thrust::unique_copy(begin, end, unique_out.begin(), equality_op);
+
+  // Ensure number of selected items are correct
+  offset_t num_selected_out = static_cast<offset_t>(cuda::std::distance(unique_out.begin(), unique_out_end));
+  ASSERT_EQUAL(num_selected_out, expected_num_unique);
+  unique_out.resize(expected_num_unique);
+
+  // Ensure selected items are correct
+  auto expected_out_it     = thrust::make_transform_iterator(begin, multiply_n<offset_t>{run_length_of_equal_items});
+  bool all_results_correct = thrust::equal(unique_out.begin(), unique_out.end(), expected_out_it);
+  ASSERT_EQUAL(all_results_correct, true);
+}
+
+void TestUniqueWithLargeNumberOfItems()
+try
+{
+  for (int mag : {30, 31, 32, 33})
+  {
+    TestUniqueWithMagnitude(mag);
+  }
+}
+catch (std::bad_alloc&)
+{
+  // if we run out of memory, just skip the test
+  return;
+}
+DECLARE_UNITTEST(TestUniqueWithLargeNumberOfItems);
+
+void TestUniqueWithCustomEqualityOp()
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  auto constexpr num_items = 1000;
+  auto data                = thrust::make_counting_iterator(T{0});
+
+  thrust::device_vector<::cuda::std::uint32_t> error_counter(1, 0);
+  auto const error_counter_ptr = thrust::raw_pointer_cast(error_counter.data());
+
+  Vector unique_out(num_items);
+  auto unique_out_end = thrust::unique_copy(
+    data, data + num_items, unique_out.begin(), check_valid_item_op{error_counter_ptr, num_items - 1});
+
+  auto num_selected_out = cuda::std::distance(unique_out.begin(), unique_out_end);
+  ASSERT_EQUAL(num_selected_out, num_items);
+  ASSERT_EQUAL(error_counter[0], ::cuda::std::uint32_t{0});
+  bool all_results_correct = thrust::equal(unique_out.cbegin(), unique_out.cend(), data);
+  ASSERT_EQUAL(all_results_correct, true);
+}
+
+DECLARE_UNITTEST(TestUniqueWithCustomEqualityOp);
+
+template <typename F>
+struct NonConstAdapter
+{
+  F f;
+  NonConstAdapter(const F& func)
+      : f(func)
+  {}
+
+  template <typename... Args>
+  __device__ auto operator()(Args&&... args) -> decltype(f(cuda::std::forward<Args>(args)...))
+  {
+    return f(cuda::std::forward<Args>(args)...);
+  }
+};
+
+void TestUniqueWithCustomEqualityOpMutable()
+{
+  using Vector = thrust::device_vector<int>;
+
+  thrust::device_vector<int> in = {1, 1, 2, 3, 4, 4, 5};
+  thrust::unique(thrust::cuda::par, in.begin(), in.end(), NonConstAdapter(cuda::std::equal_to<>{}));
+}
+
+DECLARE_UNITTEST(TestUniqueWithCustomEqualityOpMutable);

cccl_upstream/thrust/testing/cuda/unique_by_key.cu

@@ -0,0 +1,397 @@
+#include <thrust/execution_policy.h>
+#include <thrust/functional.h>
+#include <thrust/unique.h>
+
+#include <unittest/unittest.h>
+
+template <typename T>
+struct is_equal_div_10_unique
+{
+  _CCCL_HOST_DEVICE bool operator()(const T x, const T& y) const
+  {
+    return ((int) x / 10) == ((int) y / 10);
+  }
+};
+
+template <typename Vector>
+void initialize_keys(Vector& keys)
+{
+  keys.resize(9);
+  keys = {11, 11, 21, 20, 21, 21, 21, 37, 37};
+}
+
+template <typename Vector>
+void initialize_values(Vector& values)
+{
+  values.resize(9);
+  values = {0, 1, 2, 3, 4, 5, 6, 7, 8};
+}
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename Iterator3>
+__global__ void unique_by_key_kernel(
+  ExecutionPolicy exec, Iterator1 keys_first, Iterator1 keys_last, Iterator2 values_first, Iterator3 result)
+{
+  *result = thrust::unique_by_key(exec, keys_first, keys_last, values_first);
+}
+
+template <typename ExecutionPolicy, typename Iterator1, typename Iterator2, typename BinaryPredicate, typename Iterator3>
+__global__ void unique_by_key_kernel(
+  ExecutionPolicy exec,
+  Iterator1 keys_first,
+  Iterator1 keys_last,
+  Iterator2 values_first,
+  BinaryPredicate pred,
+  Iterator3 result)
+{
+  *result = thrust::unique_by_key(exec, keys_first, keys_last, values_first, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestUniqueByKeyDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector keys;
+  Vector values;
+
+  using iter_pair = cuda::std::pair<typename Vector::iterator, typename Vector::iterator>;
+  thrust::device_vector<iter_pair> new_last_vec(1);
+  iter_pair new_last;
+
+  // basic test
+  initialize_keys(keys);
+  initialize_values(values);
+
+  unique_by_key_kernel<<<1, 1>>>(exec, keys.begin(), keys.end(), values.begin(), new_last_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  new_last = new_last_vec[0];
+
+  ASSERT_EQUAL(new_last.first - keys.begin(), 5);
+  keys.erase(new_last.first, keys.end());
+  Vector keys_ref{11, 21, 20, 21, 37};
+  ASSERT_EQUAL(keys, keys_ref);
+
+  ASSERT_EQUAL(new_last.second - values.begin(), 5);
+  values.erase(new_last.second, values.end());
+  Vector values_ref{0, 2, 3, 4, 7};
+  ASSERT_EQUAL(values, values_ref);
+
+  // test BinaryPredicate
+  initialize_keys(keys);
+  initialize_values(values);
+
+  unique_by_key_kernel<<<1, 1>>>(
+    exec, keys.begin(), keys.end(), values.begin(), is_equal_div_10_unique<T>(), new_last_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  new_last = new_last_vec[0];
+
+  ASSERT_EQUAL(new_last.first - keys.begin(), 3);
+  keys.erase(new_last.first, keys.end());
+  keys_ref = {11, 21, 37};
+  ASSERT_EQUAL(keys, keys_ref);
+
+  ASSERT_EQUAL(new_last.second - values.begin(), 3);
+  values.erase(new_last.second, values.end());
+  values_ref = {0, 2, 7};
+  ASSERT_EQUAL(values, values_ref);
+}
+
+void TestUniqueByKeyDeviceSeq()
+{
+  TestUniqueByKeyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestUniqueByKeyDeviceSeq);
+
+void TestUniqueByKeyDeviceDevice()
+{
+  TestUniqueByKeyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestUniqueByKeyDeviceDevice);
+
+void TestUniqueByKeyDeviceNoSync()
+{
+  TestUniqueByKeyDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestUniqueByKeyDeviceNoSync);
+#endif
+
+template <typename ExecutionPolicy>
+void TestUniqueByKeyCudaStreams(ExecutionPolicy policy)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector keys;
+  Vector values;
+
+  using iter_pair = cuda::std::pair<Vector::iterator, Vector::iterator>;
+  iter_pair new_last;
+
+  // basic test
+  initialize_keys(keys);
+  initialize_values(values);
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  auto streampolicy = policy.on(s);
+
+  new_last = thrust::unique_by_key(streampolicy, keys.begin(), keys.end(), values.begin());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(new_last.first - keys.begin(), 5);
+  keys.erase(new_last.first, keys.end());
+  Vector keys_ref{11, 21, 20, 21, 37};
+  ASSERT_EQUAL(keys, keys_ref);
+
+  ASSERT_EQUAL(new_last.second - values.begin(), 5);
+  values.erase(new_last.second, values.end());
+  Vector values_ref{0, 2, 3, 4, 7};
+  ASSERT_EQUAL(values, values_ref);
+
+  // test BinaryPredicate
+  initialize_keys(keys);
+  initialize_values(values);
+
+  new_last = thrust::unique_by_key(streampolicy, keys.begin(), keys.end(), values.begin(), is_equal_div_10_unique<T>());
+
+  ASSERT_EQUAL(new_last.first - keys.begin(), 3);
+  keys.erase(new_last.first, keys.end());
+  keys_ref = {11, 21, 37};
+  ASSERT_EQUAL(keys, keys_ref);
+
+  ASSERT_EQUAL(new_last.second - values.begin(), 3);
+  values.erase(new_last.second, values.end());
+  values_ref = {0, 2, 7};
+  ASSERT_EQUAL(values, values_ref);
+
+  cudaStreamDestroy(s);
+}
+
+void TestUniqueByKeyCudaStreamsSync()
+{
+  TestUniqueByKeyCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestUniqueByKeyCudaStreamsSync);
+
+void TestUniqueByKeyCudaStreamsNoSync()
+{
+  TestUniqueByKeyCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestUniqueByKeyCudaStreamsNoSync);
+
+#ifdef THRUST_TEST_DEVICE_SIDE
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Iterator4,
+          typename Iterator5>
+__global__ void unique_by_key_copy_kernel(
+  ExecutionPolicy exec,
+  Iterator1 keys_first,
+  Iterator1 keys_last,
+  Iterator2 values_first,
+  Iterator3 keys_result,
+  Iterator4 values_result,
+  Iterator5 result)
+{
+  *result = thrust::unique_by_key_copy(exec, keys_first, keys_last, values_first, keys_result, values_result);
+}
+
+template <typename ExecutionPolicy,
+          typename Iterator1,
+          typename Iterator2,
+          typename Iterator3,
+          typename Iterator4,
+          typename BinaryPredicate,
+          typename Iterator5>
+__global__ void unique_by_key_copy_kernel(
+  ExecutionPolicy exec,
+  Iterator1 keys_first,
+  Iterator1 keys_last,
+  Iterator2 values_first,
+  Iterator3 keys_result,
+  Iterator4 values_result,
+  BinaryPredicate pred,
+  Iterator5 result)
+{
+  *result = thrust::unique_by_key_copy(exec, keys_first, keys_last, values_first, keys_result, values_result, pred);
+}
+
+template <typename ExecutionPolicy>
+void TestUniqueCopyByKeyDevice(ExecutionPolicy exec)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector keys;
+  Vector values;
+
+  using iter_pair = cuda::std::pair<typename Vector::iterator, typename Vector::iterator>;
+  thrust::device_vector<iter_pair> new_last_vec(1);
+  iter_pair new_last;
+
+  // basic test
+  initialize_keys(keys);
+  initialize_values(values);
+
+  Vector output_keys(keys.size());
+  Vector output_values(values.size());
+
+  unique_by_key_copy_kernel<<<1, 1>>>(
+    exec, keys.begin(), keys.end(), values.begin(), output_keys.begin(), output_values.begin(), new_last_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  new_last = new_last_vec[0];
+
+  ASSERT_EQUAL(new_last.first - output_keys.begin(), 5);
+  output_keys.erase(new_last.first, output_keys.end());
+  Vector keys_ref{11, 21, 20, 21, 37};
+  ASSERT_EQUAL(output_keys, keys_ref);
+
+  ASSERT_EQUAL(new_last.second - output_values.begin(), 5);
+  output_values.erase(new_last.second, output_values.end());
+  Vector values_ref{0, 2, 3, 4, 7};
+  ASSERT_EQUAL(output_values, values_ref);
+
+  // test BinaryPredicate
+  initialize_keys(keys);
+  initialize_values(values);
+
+  unique_by_key_copy_kernel<<<1, 1>>>(
+    exec,
+    keys.begin(),
+    keys.end(),
+    values.begin(),
+    output_keys.begin(),
+    output_values.begin(),
+    is_equal_div_10_unique<T>(),
+    new_last_vec.begin());
+  {
+    cudaError_t const err = cudaDeviceSynchronize();
+    ASSERT_EQUAL(cudaSuccess, err);
+  }
+
+  new_last = new_last_vec[0];
+
+  ASSERT_EQUAL(new_last.first - output_keys.begin(), 3);
+
+  output_keys.erase(new_last.first, output_keys.end());
+  keys_ref = {11, 21, 37};
+  ASSERT_EQUAL(output_keys, keys_ref);
+
+  ASSERT_EQUAL(new_last.second - output_values.begin(), 3);
+  output_values.erase(new_last.second, output_values.end());
+  values_ref = {0, 2, 7};
+  ASSERT_EQUAL(output_values, values_ref);
+}
+
+void TestUniqueCopyByKeyDeviceSeq()
+{
+  TestUniqueCopyByKeyDevice(thrust::seq);
+}
+DECLARE_UNITTEST(TestUniqueCopyByKeyDeviceSeq);
+
+void TestUniqueCopyByKeyDeviceDevice()
+{
+  TestUniqueCopyByKeyDevice(thrust::device);
+}
+DECLARE_UNITTEST(TestUniqueCopyByKeyDeviceDevice);
+
+void TestUniqueCopyByKeyDeviceNoSync()
+{
+  TestUniqueCopyByKeyDevice(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestUniqueCopyByKeyDeviceNoSync);
+#endif
+
+template <typename ExecutionPolicy>
+void TestUniqueCopyByKeyCudaStreams(ExecutionPolicy policy)
+{
+  using Vector = thrust::device_vector<int>;
+  using T      = Vector::value_type;
+
+  Vector keys;
+  Vector values;
+
+  using iter_pair = cuda::std::pair<Vector::iterator, Vector::iterator>;
+  iter_pair new_last;
+
+  // basic test
+  initialize_keys(keys);
+  initialize_values(values);
+
+  Vector output_keys(keys.size());
+  Vector output_values(values.size());
+
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  auto streampolicy = policy.on(s);
+
+  new_last = thrust::unique_by_key_copy(
+    streampolicy, keys.begin(), keys.end(), values.begin(), output_keys.begin(), output_values.begin());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(new_last.first - output_keys.begin(), 5);
+  output_keys.erase(new_last.first, output_keys.end());
+  Vector keys_ref{11, 21, 20, 21, 37};
+  ASSERT_EQUAL(output_keys, keys_ref);
+
+  ASSERT_EQUAL(new_last.second - output_values.begin(), 5);
+  output_values.erase(new_last.second, output_values.end());
+  Vector values_ref{0, 2, 3, 4, 7};
+  ASSERT_EQUAL(output_values, values_ref);
+
+  // test BinaryPredicate
+  initialize_keys(keys);
+  initialize_values(values);
+
+  new_last = thrust::unique_by_key_copy(
+    streampolicy,
+    keys.begin(),
+    keys.end(),
+    values.begin(),
+    output_keys.begin(),
+    output_values.begin(),
+    is_equal_div_10_unique<T>());
+  cudaStreamSynchronize(s);
+
+  ASSERT_EQUAL(new_last.first - output_keys.begin(), 3);
+  output_keys.erase(new_last.first, output_keys.end());
+  keys_ref = {11, 21, 37};
+  ASSERT_EQUAL(output_keys, keys_ref);
+
+  ASSERT_EQUAL(new_last.second - output_values.begin(), 3);
+  output_values.erase(new_last.second, output_values.end());
+  values_ref = {0, 2, 7};
+  ASSERT_EQUAL(output_values, values_ref);
+
+  cudaStreamDestroy(s);
+}
+
+void TestUniqueCopyByKeyCudaStreamsSync()
+{
+  TestUniqueCopyByKeyCudaStreams(thrust::cuda::par);
+}
+DECLARE_UNITTEST(TestUniqueCopyByKeyCudaStreamsSync);
+
+void TestUniqueCopyByKeyCudaStreamsNoSync()
+{
+  TestUniqueCopyByKeyCudaStreams(thrust::cuda::par_nosync);
+}
+DECLARE_UNITTEST(TestUniqueCopyByKeyCudaStreamsNoSync);