[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/c/parallel/test/CMakeLists.txt

@@ -0,0 +1,56 @@
+cccl_get_c2h()
+
+function(cccl_c_parallel_add_test target_name_var source)
+  get_filename_component(target_name "${source}" NAME_WE)
+  string(
+    REGEX REPLACE
+    "test_([^.]*)"
+    "cccl.c.parallel.test.\\1"
+    target_name
+    "${target_name}"
+  )
+  set(target_name_var ${target_name} PARENT_SCOPE)
+
+  cccl_add_executable(
+    ${target_name}
+    ADD_CTEST
+    NO_METATARGETS
+    DIALECT 20
+    SOURCES "${source}"
+  )
+
+  set_target_properties(${target_name} PROPERTIES CUDA_RUNTIME_LIBRARY STATIC)
+  target_link_libraries(
+    ${target_name}
+    PRIVATE
+      cccl.compiler_interface
+      cccl.c.parallel
+      CUDA::cudart_static
+      CUDA::nvrtc
+      cccl.c2h.main
+  )
+
+  # Get the first CUDA include directory only
+  list(GET CUDAToolkit_INCLUDE_DIRS 0 CUDA_FIRST_INCLUDE_DIR)
+  target_compile_definitions(
+    ${target_name}
+    PRIVATE
+      TEST_CUB_PATH="-I${CCCL_SOURCE_DIR}/cub"
+      TEST_THRUST_PATH="-I${CCCL_SOURCE_DIR}/thrust"
+      TEST_LIBCUDACXX_PATH="-I${CCCL_SOURCE_DIR}/libcudacxx/include"
+      TEST_CTK_PATH="-I${CUDA_FIRST_INCLUDE_DIR}"
+      TEST_INCLUDE_PATH="${CMAKE_CURRENT_SOURCE_DIR}"
+  )
+endfunction()
+
+file(
+  GLOB test_srcs
+  RELATIVE "${CMAKE_CURRENT_LIST_DIR}"
+  CONFIGURE_DEPENDS
+  *.cu
+  *.cpp
+)
+
+foreach (test_src IN LISTS test_srcs)
+  cccl_c_parallel_add_test(test_target "${test_src}")
+endforeach()

cccl_upstream/c/parallel/test/algorithm_execution.h

@@ -0,0 +1,196 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <iostream>
+#include <optional>
+#include <string>
+
+#include <cuda.h>
+
+#include "test_util.h"
+#include <c2h/catch2_test_helper.h>
+#include <cccl/c/types.h>
+
+inline constexpr bool check_ldl_stl_in_sass = false;
+
+template <int device_id_ = 0>
+class BuildInformation
+{
+  int cc_major;
+  int cc_minor;
+  const char* cub_path;
+  const char* thrust_path;
+  const char* libcudacxx_path;
+  const char* ctk_path;
+
+  BuildInformation() = default;
+  BuildInformation(int major, int minor, const char* cub, const char* thrust, const char* libcudacxx, const char* ctk)
+      : cc_major(major)
+      , cc_minor(minor)
+      , cub_path(cub)
+      , thrust_path(thrust)
+      , libcudacxx_path(libcudacxx)
+      , ctk_path(ctk)
+  {}
+
+public:
+  static constexpr int device_id = device_id_;
+
+  static const auto& init()
+  {
+    cudaDeviceProp deviceProp;
+    cudaGetDeviceProperties(&deviceProp, device_id);
+
+    static BuildInformation singleton{
+      deviceProp.major, deviceProp.minor, TEST_CUB_PATH, TEST_THRUST_PATH, TEST_LIBCUDACXX_PATH, TEST_CTK_PATH};
+    return singleton;
+  }
+
+  int get_cc_major() const
+  {
+    return cc_major;
+  }
+  int get_cc_minor() const
+  {
+    return cc_minor;
+  }
+  const char* get_cub_path() const
+  {
+    return cub_path;
+  }
+  const char* get_thrust_path() const
+  {
+    return thrust_path;
+  }
+  const char* get_libcudacxx_path() const
+  {
+    return libcudacxx_path;
+  }
+  const char* get_ctk_path() const
+  {
+    return ctk_path;
+  }
+};
+
+template <typename Build, typename = void>
+struct build_traits
+{
+  static bool should_check_sass(int)
+  {
+    return true;
+  }
+};
+
+template <typename Build>
+struct build_traits<Build, std::void_t<decltype(Build::should_check_sass(0))>>
+{
+  static bool should_check_sass(int cc_major)
+  {
+    return Build::should_check_sass(cc_major);
+  }
+};
+
+template <typename BuildResultT,
+          typename Build,
+          typename Cleanup,
+          typename Run,
+          typename BuildCache,
+          typename KeyT,
+          typename... Tx>
+void AlgorithmExecute(std::optional<BuildCache>& cache, const std::optional<KeyT>& lookup_key, Tx&&... args)
+{
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  BuildResultT build{};
+
+  bool found               = false;
+  const bool cache_and_key = bool(cache) && bool(lookup_key);
+
+  if (cache_and_key)
+  {
+    auto& cache_v     = cache.value(); // NOLINT(bugprone-unchecked-optional-access)
+    const auto& key_v = lookup_key.value(); // NOLINT(bugprone-unchecked-optional-access)
+    if (cache_v.contains(key_v))
+    {
+      build = cache_v.get(key_v).get();
+      found = true;
+    }
+  }
+
+  if (!found)
+  {
+    REQUIRE(
+      CUDA_SUCCESS
+      == Build{}(&build,
+                 args...,
+                 build_info.get_cc_major(),
+                 build_info.get_cc_minor(),
+                 build_info.get_cub_path(),
+                 build_info.get_thrust_path(),
+                 build_info.get_libcudacxx_path(),
+                 build_info.get_ctk_path()));
+
+    if (cache_and_key)
+    {
+      auto& cache_v     = cache.value(); // NOLINT(bugprone-unchecked-optional-access)
+      const auto& key_v = lookup_key.value(); // NOLINT(bugprone-unchecked-optional-access)
+      cache_v.insert(key_v, build);
+    }
+  }
+
+  if constexpr (check_ldl_stl_in_sass && build_traits<Build>::should_check_sass(build_info.get_cc_major()))
+  {
+    const std::string sass = inspect_sass(build.payload, build.payload_size);
+    REQUIRE(sass.find("LDL") == std::string::npos);
+    REQUIRE(sass.find("STL") == std::string::npos);
+  }
+
+  CUstream null_stream = nullptr;
+
+  size_t temp_storage_bytes = 0;
+  REQUIRE(CUDA_SUCCESS == Run{}(build, nullptr, &temp_storage_bytes, args..., null_stream));
+
+  pointer_t<uint8_t> temp_storage(temp_storage_bytes);
+
+  REQUIRE(CUDA_SUCCESS == Run{}(build, temp_storage.ptr, &temp_storage_bytes, args..., null_stream));
+
+  if (cache_and_key)
+  {
+    // if cache and lookup_key were provided, the ownership of resources
+    // allocated for build is transferred to the cache, hence do nothing
+  }
+  else
+  {
+    // release build data resources
+    REQUIRE(CUDA_SUCCESS == Cleanup{}(&build));
+  }
+}
+
+template <typename BuildResultT, typename Cleanup>
+struct BuildResultDeleter
+{
+  static constexpr Cleanup cleanup_{};
+  void operator()(BuildResultT* build_data) const noexcept
+  {
+    BuildResultDeleter::check_success(cleanup_(build_data));
+  }
+
+private:
+  static void check_success(CUresult status) noexcept
+  {
+    if (status != CUDA_SUCCESS)
+    {
+      std::cerr << "Clean-up call returned status " << status << '\n';
+    }
+  }
+};

cccl_upstream/c/parallel/test/build_result_caching.h

@@ -0,0 +1,174 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <cassert>
+#include <iostream>
+#include <memory>
+#include <optional>
+#include <sstream>
+#include <tuple>
+#include <typeinfo>
+#include <unordered_map>
+
+template <typename ResultT, typename CleanupCallable>
+class result_wrapper_t
+{
+  std::shared_ptr<ResultT> m_owner;
+
+public:
+  result_wrapper_t()
+      : m_owner{}
+  {}
+  result_wrapper_t(ResultT v)
+      : m_owner{std::make_shared<ResultT>(v)}
+  {}
+
+  result_wrapper_t(const result_wrapper_t&) = default;
+  result_wrapper_t(result_wrapper_t&&)      = default;
+
+  result_wrapper_t& operator=(const result_wrapper_t&) = default;
+  result_wrapper_t& operator=(result_wrapper_t&&)      = default;
+
+  ~result_wrapper_t() noexcept
+  try
+  {
+    if (!m_owner)
+    {
+      return;
+    }
+
+    if (m_owner.use_count() <= 1)
+    {
+      // release resources
+      CleanupCallable{}(m_owner.get());
+    }
+  }
+  catch (const std::exception& e)
+  {
+    std::cerr << "~result_wrapper_t ignores exception: " << e.what() << '\n';
+  }
+
+  ResultT& get()
+  {
+    return *m_owner.get();
+  }
+};
+
+template <typename KeyT, typename ValueT>
+class build_cache_t
+{
+  std::unordered_map<KeyT, ValueT> m_map{};
+
+public:
+  build_cache_t() = default;
+
+  bool contains(const KeyT& key) const
+  {
+    // unorder_map::contains is C++20 feature
+    return m_map.contains(key);
+  }
+
+  void insert(const KeyT& key, ValueT&& new_value)
+  {
+    m_map[key] = std::move(new_value);
+  }
+
+  ValueT& get(const KeyT& key)
+  {
+    assert(m_map.contains(key));
+    return m_map[key];
+  }
+};
+
+template <typename T, typename Tag>
+class fixture
+{
+public:
+  using OptionalT = typename std::optional<T>;
+
+private:
+  OptionalT v;
+
+  fixture()
+      : v{T{}}
+  {}
+
+public:
+  OptionalT& get_value()
+  {
+    return v;
+  }
+
+  static auto& get_or_create()
+  {
+    static fixture singleton{};
+    return singleton;
+  }
+};
+
+struct KeyBuilder
+{
+  static std::string bool_as_key(bool v)
+  {
+    return (v) ? std::string("T") : std::string("F");
+  }
+
+  template <typename T>
+  static std::string type_as_key()
+  {
+    return typeid(T).name();
+  }
+
+  template <std::size_t N>
+  static std::string join(const std::string (&collection)[N])
+  {
+    constexpr std::string_view delimiter = "-";
+    std::stringstream ss;
+
+    for (std::size_t i = 0; i < N; ++i)
+    {
+      ss << collection[i];
+      if (i + 1 < N)
+      {
+        ss << delimiter;
+      }
+    }
+
+    return ss.str();
+  }
+};
+
+template <typename TupleLike, std::size_t I = 0>
+void adder_helper(std::stringstream& ss)
+{
+  constexpr std::size_t S = std::tuple_size_v<TupleLike>;
+  if constexpr (I < S)
+  {
+    using SelectedType       = std::tuple_element_t<I, TupleLike>;
+    constexpr std::size_t In = I + 1;
+
+    ss << KeyBuilder::type_as_key<SelectedType>();
+    if constexpr (In < S)
+    {
+      ss << "-";
+    }
+    adder_helper<TupleLike, In>(ss);
+  }
+}
+
+template <typename... Ts>
+std::optional<std::string> make_key()
+{
+  std::stringstream ss{};
+  adder_helper<std::tuple<Ts...>, 0>(ss);
+  return std::make_optional(ss.str());
+}

cccl_upstream/c/parallel/test/test_binary_search.cpp

@@ -0,0 +1,336 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <string>
+
+#include <cuda_runtime.h>
+
+#include "algorithm_execution.h"
+#include "build_result_caching.h"
+#include "test_util.h"
+#include <cccl/c/binary_search.h>
+
+using BuildResultT = cccl_device_binary_search_build_result_t;
+
+struct binary_search_cleanup
+{
+  CUresult operator()(BuildResultT* build_data) const noexcept
+  {
+    return cccl_device_binary_search_cleanup(build_data);
+  }
+};
+
+static std::string mode_as_key(cccl_binary_search_mode_t mode)
+{
+  switch (mode)
+  {
+    case cccl_binary_search_mode_t::CCCL_BINARY_SEARCH_LOWER_BOUND:
+      return "LOWER";
+    case cccl_binary_search_mode_t::CCCL_BINARY_SEARCH_UPPER_BOUND:
+      return "UPPER";
+  }
+
+  throw std::runtime_error("Invalid binary search mode");
+}
+
+template <typename T>
+std::optional<std::string> make_binary_search_key(bool inclusive, cccl_binary_search_mode_t mode)
+{
+  const std::string parts[] = {KeyBuilder::type_as_key<T>(), KeyBuilder::bool_as_key(inclusive), mode_as_key(mode)};
+  return KeyBuilder::join(parts);
+}
+
+using binary_search_deleter       = BuildResultDeleter<BuildResultT, binary_search_cleanup>;
+using binary_search_build_cache_t = build_cache_t<std::string, result_wrapper_t<BuildResultT, binary_search_deleter>>;
+
+template <typename Tag>
+auto& get_cache()
+{
+  return fixture<binary_search_build_cache_t, Tag>::get_or_create().get_value();
+}
+
+struct binary_search_build
+{
+  CUresult operator()(
+    BuildResultT* build_ptr,
+    cccl_binary_search_mode_t mode,
+    cccl_iterator_t data,
+    uint64_t,
+    cccl_iterator_t values,
+    uint64_t,
+    cccl_iterator_t out,
+    cccl_op_t op,
+    int cc_major,
+    int cc_minor,
+    const char* cub_path,
+    const char* thrust_path,
+    const char* libcudacxx_path,
+    const char* ctk_path) const noexcept
+  {
+    return cccl_device_binary_search_build(
+      build_ptr, mode, data, values, out, op, cc_major, cc_minor, cub_path, thrust_path, libcudacxx_path, ctk_path);
+  }
+
+  static constexpr bool should_check_sass(int)
+  {
+    return false;
+  }
+};
+
+struct binary_search_run
+{
+  template <typename... Ts>
+  CUresult operator()(
+    BuildResultT build, void* scratch, std::size_t* scratch_size, cccl_binary_search_mode_t, Ts... args) const noexcept
+  {
+    *scratch_size = 1;
+    return (scratch) ? cccl_device_binary_search(build, args...) : CUDA_SUCCESS;
+  }
+};
+
+template <cccl_binary_search_mode_t Mode>
+struct binary_search_wrapper
+{
+  static const constexpr auto mode = Mode;
+
+  template <typename BuildCache = binary_search_build_cache_t, typename KeyT = std::string>
+  void operator()(
+    cccl_iterator_t data,
+    uint64_t num_items,
+    cccl_iterator_t values,
+    uint64_t num_values,
+    cccl_iterator_t output,
+    cccl_op_t op,
+    std::optional<BuildCache>& cache,
+    const std::optional<KeyT>& lookup_key) const
+  {
+    AlgorithmExecute<BuildResultT, binary_search_build, binary_search_cleanup, binary_search_run, BuildCache, KeyT>(
+      cache, lookup_key, mode, data, num_items, values, num_values, output, op);
+  }
+};
+
+using lower_bound = binary_search_wrapper<cccl_binary_search_mode_t::CCCL_BINARY_SEARCH_LOWER_BOUND>;
+using upper_bound = binary_search_wrapper<cccl_binary_search_mode_t::CCCL_BINARY_SEARCH_UPPER_BOUND>;
+
+// ==============
+//   Test section
+// ==============
+
+using integral_types = c2h::type_list<int32_t, uint32_t, int64_t, uint64_t>;
+
+struct std_lower_bound_t
+{
+  template <typename RangeIteratorT, typename T, typename CompareOpT>
+  RangeIteratorT operator()(RangeIteratorT first, RangeIteratorT last, const T& value, CompareOpT comp) const
+  {
+    return std::lower_bound(first, last, value, comp);
+  }
+} std_lower_bound;
+
+struct std_upper_bound_t
+{
+  template <typename RangeIteratorT, typename T, typename CompareOpT>
+  RangeIteratorT operator()(RangeIteratorT first, RangeIteratorT last, const T& value, CompareOpT comp) const
+  {
+    return std::upper_bound(first, last, value, comp);
+  }
+} std_upper_bound;
+
+template <typename Fixture, typename Value, typename Variant, typename HostVariant>
+void test_vectorized(Variant variant, HostVariant host_variant)
+{
+  const std::size_t num_items = GENERATE(0, 43, take(4, random(1 << 12, 1 << 16)));
+  operation_t op              = make_operation("op", get_merge_sort_op(get_type_info<Value>().type));
+
+  const std::vector<Value> target_values = generate<Value>(num_items / 100);
+  std::vector<Value> data                = generate<Value>(num_items);
+  std::copy(target_values.begin(), target_values.end(), data.begin());
+  std::sort(data.begin(), data.end());
+  const std::vector<std::ptrdiff_t> output(target_values.size(), 0);
+
+  pointer_t<Value> target_values_ptr(target_values);
+  pointer_t<Value> data_ptr(data);
+  pointer_t<std::ptrdiff_t> output_ptr(output);
+
+  auto& build_cache    = get_cache<Fixture>();
+  const auto& test_key = make_binary_search_key<Value>(true, Variant::mode);
+
+  variant(data_ptr, num_items, target_values_ptr, target_values.size(), output_ptr, op, build_cache, test_key);
+
+  std::vector<std::ptrdiff_t> results(output_ptr);
+  std::vector<std::ptrdiff_t> expected(target_values.size(), 0);
+
+  std::vector<std::ptrdiff_t> expected_results(target_values.size(), 0);
+
+  for (auto i = 0u; i < target_values.size(); ++i)
+  {
+    expected_results[i] =
+      host_variant(data.data(), data.data() + num_items, target_values[i], std::less<>()) - data.data();
+  }
+
+  CHECK(expected_results == results);
+}
+
+struct BinarySearch_IntegralTypes_LowerBound_Fixture_Tag;
+C2H_TEST("DeviceFind::LowerBound works", "[find][device][binary-search]", integral_types)
+{
+  using value_type = c2h::get<0, TestType>;
+  test_vectorized<BinarySearch_IntegralTypes_LowerBound_Fixture_Tag, value_type>(lower_bound{}, std_lower_bound);
+}
+
+struct BinarySearch_IntegralTypes_UpperBound_Fixture_Tag;
+C2H_TEST("DeviceFind::UpperBound works", "[find][device][binary-search]", integral_types)
+{
+  using value_type = c2h::get<0, TestType>;
+  test_vectorized<BinarySearch_IntegralTypes_UpperBound_Fixture_Tag, value_type>(upper_bound{}, std_upper_bound);
+}
+
+#ifndef CCCL_C_PARALLEL_V2
+C2H_TEST("BinarySearch build result has serialization metadata populated", "[binary_search][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  cccl_op_t op = make_well_known_less_binary_predicate();
+  pointer_t<T> data(1);
+  pointer_t<T> values(1);
+  pointer_t<T> out(1);
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_binary_search_build(
+      &build,
+      CCCL_BINARY_SEARCH_LOWER_BOUND,
+      data,
+      values,
+      out,
+      op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path()));
+
+  CHECK(build.transform.cc == build_info.get_cc_major() * 10 + build_info.get_cc_minor());
+  CHECK((build.transform.payload != nullptr && build.transform.payload_kind == CCCL_PAYLOAD_CUBIN));
+  CHECK(build.transform.payload_size > 0);
+  REQUIRE(build.transform.transform_kernel_lowered_name != nullptr);
+  CHECK(build.transform.transform_kernel_lowered_name[0] != '\0');
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_binary_search_cleanup(&build));
+}
+
+C2H_TEST("BinarySearch compile/load round-trip", "[binary_search][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  operation_t op = make_operation("op", get_merge_sort_op(get_type_info<T>().type));
+  pointer_t<T> dummy_data(1);
+  pointer_t<T> dummy_values(1);
+  pointer_t<std::ptrdiff_t> dummy_out(1);
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_binary_search_compile(
+      &build,
+      CCCL_BINARY_SEARCH_LOWER_BOUND,
+      dummy_data,
+      dummy_values,
+      dummy_out,
+      op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+
+  REQUIRE((build.transform.payload != nullptr && build.transform.payload_kind == CCCL_PAYLOAD_CUBIN));
+  REQUIRE(build.transform.payload_size > 0);
+  REQUIRE(build.transform.transform_kernel_lowered_name != nullptr);
+  CHECK(build.transform.library == nullptr);
+  CHECK(build.transform.transform_kernel == nullptr);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_binary_search_load(&build));
+  REQUIRE(build.transform.library != nullptr);
+  CHECK(build.transform.transform_kernel != nullptr);
+
+  constexpr std::size_t n_items  = 16;
+  constexpr std::size_t n_values = 4;
+  std::vector<T> data            = generate<T>(n_items);
+  std::sort(data.begin(), data.end());
+  const std::vector<T> values = generate<T>(n_values);
+  pointer_t<T> data_ptr(data);
+  pointer_t<T> values_ptr(values);
+  pointer_t<std::ptrdiff_t> output_ptr(n_values);
+  CUstream null_stream = nullptr;
+
+  REQUIRE(CUDA_SUCCESS
+          == cccl_device_binary_search(build, data_ptr, n_items, values_ptr, n_values, output_ptr, op, null_stream));
+
+  std::vector<std::ptrdiff_t> expected(n_values);
+  for (std::size_t i = 0; i < n_values; ++i)
+  {
+    expected[i] = std::lower_bound(data.begin(), data.end(), values[i]) - data.begin();
+  }
+  REQUIRE(expected == std::vector<std::ptrdiff_t>(output_ptr));
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_binary_search_cleanup(&build));
+}
+
+C2H_TEST("BinarySearch compile rejects kernel-only comparator op", "[binary_search][serialization]")
+{
+  using T                 = int32_t;
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  // Kernel-only op: code_size == 0 with a non-empty name.
+  // binary_search wraps the comparator in a generated function, so the wrapper
+  // cannot be decoupled from the comparator type at link time.
+  cccl_op_t custom_op{};
+  custom_op.type      = CCCL_STATELESS;
+  custom_op.name      = "my_comparator";
+  custom_op.code_size = 0;
+  custom_op.code_type = CCCL_OP_LTOIR;
+  custom_op.size      = 1;
+  custom_op.alignment = 1;
+
+  pointer_t<T> dummy_data(1);
+  pointer_t<T> dummy_values(1);
+  pointer_t<std::ptrdiff_t> dummy_out(1);
+
+  cccl_device_binary_search_build_result_t build{};
+  REQUIRE(
+    CUDA_ERROR_INVALID_VALUE
+    == cccl_device_binary_search_compile(
+      &build,
+      CCCL_BINARY_SEARCH_LOWER_BOUND,
+      dummy_data,
+      dummy_values,
+      dummy_out,
+      custom_op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+}
+#endif // CCCL_C_PARALLEL_V2

cccl_upstream/c/parallel/test/test_for.cpp

@@ -0,0 +1,485 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <algorithm>
+#include <cstdint>
+#include <iostream> // std::cerr
+#include <optional> // std::optional
+#include <string>
+
+#include <cuda_runtime.h>
+
+#include "algorithm_execution.h"
+#include "build_result_caching.h"
+#include "test_util.h"
+#include <cccl/c/for.h>
+
+using BuildResultT = cccl_device_for_build_result_t;
+
+struct for_each_cleanup
+{
+  CUresult operator()(BuildResultT* build_data) const noexcept
+  {
+    return cccl_device_for_cleanup(build_data);
+  }
+};
+
+using for_each_deleter       = BuildResultDeleter<BuildResultT, for_each_cleanup>;
+using for_each_build_cache_t = build_cache_t<std::string, result_wrapper_t<BuildResultT, for_each_deleter>>;
+
+struct for_each_build
+{
+  template <typename... Ts>
+  CUresult operator()(BuildResultT* build_ptr, cccl_iterator_t input, uint64_t, cccl_op_t op, Ts... args) const noexcept
+  {
+    return cccl_device_for_build(build_ptr, input, op, args...);
+  }
+};
+
+struct for_each_run
+{
+  template <typename... Ts>
+  CUresult operator()(BuildResultT build, void* scratch, size_t* nbytes, Ts... args) const noexcept
+  {
+    *nbytes = 1;
+    // only run if scratch is not null
+    return (scratch) ? cccl_device_for(build, args...) : CUDA_SUCCESS;
+  }
+};
+
+template <typename BuildCache = for_each_build_cache_t, typename KeyT = std::string>
+void for_each(cccl_iterator_t input,
+              uint64_t num_items,
+              cccl_op_t op,
+              std::optional<BuildCache>& cache,
+              const std::optional<KeyT>& lookup_key)
+{
+  AlgorithmExecute<BuildResultT, for_each_build, for_each_cleanup, for_each_run, BuildCache, KeyT>(
+    cache, lookup_key, input, num_items, op);
+}
+
+// Specialization for a pointer input
+struct DeviceFor_Pointer_Fixture_Tag;
+
+template <typename T>
+void for_each_pointer_input(pointer_t<T>& input_ptr, uint64_t num_items, cccl_op_t op)
+{
+  auto& build_cache    = fixture<for_each_build_cache_t, DeviceFor_Pointer_Fixture_Tag>::get_or_create().get_value();
+  const auto& test_key = make_key<T>();
+
+  for_each(static_cast<cccl_iterator_t>(input_ptr), num_items, op, build_cache, test_key);
+}
+
+// specialization without caching
+void for_each_uncached(cccl_iterator_t input, uint64_t num_items, cccl_op_t op)
+{
+  std::optional<for_each_build_cache_t> no_cache = std::nullopt;
+  std::optional<std::string> no_key              = std::nullopt;
+
+  for_each(input, num_items, op, no_cache, no_key);
+}
+
+using integral_types = c2h::type_list<int32_t, uint32_t, int64_t, uint64_t>;
+C2H_TEST("for works with integral types", "[for]", integral_types)
+{
+  using T = c2h::get<0, TestType>;
+
+  const uint64_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 24)));
+
+  operation_t op = make_operation("op", get_for_op(get_type_info<T>().type));
+  std::vector<T> input(num_items, T(1));
+  pointer_t<T> input_ptr(input);
+
+  for_each_pointer_input(input_ptr, num_items, op);
+
+  // Copy input array back to host
+  input = input_ptr;
+
+  REQUIRE(std::all_of(input.begin(), input.end(), [](auto&& v) {
+    return v == T{2};
+  }));
+}
+
+struct pair
+{
+  short a;
+  size_t b;
+};
+
+C2H_TEST("for works with custom types", "[for]")
+{
+  const int num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 24)));
+
+  operation_t op = make_operation("op",
+                                  R"XXX(
+struct pair { short a; size_t b; };
+extern "C" __device__ void op(void* a_ptr) {
+  pair* a = static_cast<pair*>(a_ptr);
+  a->a++;
+  a->b++;
+}
+)XXX");
+
+  std::vector<pair> input(num_items, pair{short(1), size_t(1)});
+  pointer_t<pair> input_ptr(input);
+
+  for_each_pointer_input(input_ptr, num_items, op);
+
+  // Copy back input array
+  input = input_ptr;
+
+  REQUIRE(std::all_of(input.begin(), input.end(), [](auto v) {
+    return (v.a == short(2)) && (v.b == size_t(2));
+  }));
+}
+
+struct invocation_counter_state_t
+{
+  int* d_counter;
+};
+
+C2H_TEST("for_each works with stateful operators", "[for_each]")
+{
+  const int num_items = 1 << 12;
+  pointer_t<int> counter(1);
+  invocation_counter_state_t op_state                 = {counter.ptr};
+  stateful_operation_t<invocation_counter_state_t> op = make_operation(
+    "op",
+    R"XXX(
+struct invocation_counter_state_t { int* d_counter; };
+extern "C" __device__ void op(void* state_ptr, void* a_ptr) {
+  invocation_counter_state_t* state = static_cast<invocation_counter_state_t*>(state_ptr);
+  atomicAdd(state->d_counter, *static_cast<int*>(a_ptr));
+}
+)XXX",
+    op_state);
+
+  std::vector<int> input(num_items, 1);
+  pointer_t<int> input_ptr(input);
+
+  for_each_uncached(input_ptr, num_items, op);
+
+  const int invocation_count = counter[0];
+  REQUIRE(invocation_count == num_items);
+}
+
+struct large_state_t
+{
+  int x;
+  int* d_counter;
+  int y, z, a;
+};
+
+C2H_TEST("for_each works with large stateful operators", "[for_each]")
+{
+  const int num_items = 1 << 12;
+  pointer_t<int> counter(1);
+  large_state_t op_state                 = {1, counter.ptr, 2, 3, 4};
+  stateful_operation_t<large_state_t> op = make_operation(
+    "op",
+    R"XXX(
+struct large_state_t
+{
+  int x;
+  int* d_counter;
+  int y, z, a;
+};
+extern "C" __device__ void op(void* state_ptr, void* a_ptr) {
+  large_state_t* state = static_cast<large_state_t*>(state_ptr);
+  atomicAdd(state->d_counter, *static_cast<int*>(a_ptr));
+}
+)XXX",
+    op_state);
+
+  std::vector<int> input(num_items, 1);
+  pointer_t<int> input_ptr(input);
+
+  for_each_uncached(input_ptr, num_items, op);
+
+  const int invocation_count = counter[0];
+  REQUIRE(invocation_count == num_items);
+}
+
+C2H_TEST("for works with C++ source operations", "[for]")
+{
+  using T = int32_t;
+
+  const uint64_t num_items = GENERATE(42, 1337, 42000);
+
+  // Create operation from C++ source instead of LTO-IR
+  std::string cpp_source = R"(
+    extern "C" __device__ void op(void* a) {
+      int* ia = (int*)a;
+      *ia = *ia + 1;
+    }
+  )";
+
+  operation_t op = make_cpp_operation("op", cpp_source);
+
+  std::vector<T> input(num_items, T(1));
+  pointer_t<T> input_ptr(input);
+
+  // Test key including flag that this uses C++ source
+  std::optional<std::string> test_key = std::format("cpp_source_test_{}_{}", num_items, typeid(T).name());
+
+  auto& cache = fixture<for_each_build_cache_t, DeviceFor_Pointer_Fixture_Tag>::get_or_create().get_value();
+  std::optional<for_each_build_cache_t> cache_opt = cache;
+  for_each(input_ptr, num_items, op, cache_opt, test_key);
+
+  // Copy input array back to host
+  input = input_ptr;
+
+  REQUIRE(std::all_of(input.begin(), input.end(), [](auto&& v) {
+    return v == T{2};
+  }));
+}
+
+C2H_TEST("For works with C++ source operations using custom headers", "[for]")
+{
+  using T = int32_t;
+
+  const uint64_t num_items = GENERATE(42, 1337, 42000);
+
+  // Create operation from C++ source that uses the identity function from header
+  std::string cpp_source = R"(
+    #include "test_identity.h"
+    extern "C" __device__ void op(void* a) {
+      int* ia = (int*)a;
+      int val = test_identity(*ia);
+      *ia = val + 1;
+    }
+  )";
+
+  operation_t op = make_cpp_operation("op", cpp_source);
+
+  std::vector<T> input(num_items, T(1));
+  pointer_t<T> input_ptr(input);
+
+  // Test _ex version with custom build configuration
+  const char* extra_flags[] = {"-DTEST_IDENTITY_ENABLED"};
+  const char* extra_dirs[]  = {TEST_INCLUDE_PATH};
+  cccl_build_config config  = make_build_config(extra_flags, 1, extra_dirs, 1);
+
+  // Build with _ex version
+  cccl_device_for_build_result_t build{};
+  const auto& build_info = BuildInformation<>::init();
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_for_build_ex(
+      &build,
+      input_ptr,
+      op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      &config));
+
+  // Execute the for_each
+  REQUIRE(CUDA_SUCCESS == cccl_device_for(build, input_ptr, num_items, op, CU_STREAM_LEGACY));
+
+  // Verify results
+  std::vector<T> output(num_items);
+  cudaMemcpy(output.data(), static_cast<void*>(input_ptr.ptr), sizeof(T) * num_items, cudaMemcpyDeviceToHost);
+  std::vector<T> expected = input;
+  std::transform(expected.begin(), expected.end(), expected.begin(), [](T x) {
+    return x * 2;
+  });
+  REQUIRE(output == expected);
+
+  // Cleanup
+  REQUIRE(CUDA_SUCCESS == cccl_device_for_cleanup(&build));
+}
+
+#ifndef CCCL_C_PARALLEL_V2
+C2H_TEST("For build result has serialization metadata populated", "[for][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  operation_t op = make_operation("op", get_for_op(get_type_info<T>().type));
+  pointer_t<T> input_ptr(1);
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_for_build(
+      &build,
+      input_ptr,
+      op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path()));
+
+  CHECK(build.cc == build_info.get_cc_major() * 10 + build_info.get_cc_minor());
+  CHECK((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  CHECK(build.payload_size > 0);
+  REQUIRE(build.static_kernel_lowered_name != nullptr);
+  CHECK(build.static_kernel_lowered_name[0] != '\0');
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_for_cleanup(&build));
+}
+
+C2H_TEST("For compile/load round-trip", "[for][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+  constexpr std::size_t n = 16;
+  const std::vector<T> input_h(n, T{1});
+
+  operation_t op = make_operation("op", get_for_op(get_type_info<T>().type));
+  pointer_t<T> input_ptr(input_h);
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_for_compile(
+      &build,
+      input_ptr,
+      op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+
+  REQUIRE((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  REQUIRE(build.payload_size > 0);
+  REQUIRE(build.static_kernel_lowered_name != nullptr);
+  CHECK(build.library == nullptr);
+  CHECK(build.static_kernel == nullptr);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_for_load(&build));
+  REQUIRE(build.library != nullptr);
+  CHECK(build.static_kernel != nullptr);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_for(build, input_ptr, n, op, CU_STREAM_LEGACY));
+
+  std::vector<T> output(n);
+  cudaMemcpy(output.data(), input_ptr.ptr, sizeof(T) * n, cudaMemcpyDeviceToHost);
+  REQUIRE(std::all_of(output.begin(), output.end(), [](T v) {
+    return v == T{2};
+  }));
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_for_cleanup(&build));
+}
+
+C2H_TEST("For link_ltoir round-trip", "[for][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  // Kernel-only compile: op has a name but no LTOIR (code_size == 0).
+  // compile() will produce kernel LTOIR with an unresolved external reference to "op".
+  cccl_op_t op_ko{};
+  op_ko.type      = CCCL_STATELESS;
+  op_ko.name      = "op";
+  op_ko.code      = nullptr;
+  op_ko.code_size = 0;
+  op_ko.code_type = CCCL_OP_LTOIR;
+  op_ko.size      = 1;
+  op_ko.alignment = 1;
+
+  constexpr std::size_t n = 16;
+  const std::vector<T> input_h(n, T{1});
+  pointer_t<T> input_ptr(input_h);
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_for_compile(
+      &build,
+      input_ptr,
+      op_ko,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+
+  // After kernel-only compile: payload is kernel LTOIR, not a cubin.
+  REQUIRE((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_LTOIR));
+  REQUIRE(build.payload_size > 0);
+  CHECK(build.library == nullptr);
+
+  // Compile the operator LTOIR separately (user-supplied blob).
+  operation_t op_full = make_operation("op", get_for_op(get_type_info<T>().type));
+  const void* op_blob = op_full.code.data();
+  size_t op_size      = op_full.code.size();
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_for_link_ltoir(&build, &op_blob, &op_size, 1));
+  REQUIRE((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  REQUIRE(build.library == nullptr);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_for_load(&build));
+  REQUIRE(build.library != nullptr);
+  CHECK(build.static_kernel != nullptr);
+
+  cccl_op_t op_run = op_full;
+  REQUIRE(CUDA_SUCCESS == cccl_device_for(build, input_ptr, n, op_run, CU_STREAM_LEGACY));
+
+  std::vector<T> output(n);
+  cudaMemcpy(output.data(), input_ptr.ptr, sizeof(T) * n, cudaMemcpyDeviceToHost);
+  REQUIRE(std::all_of(output.begin(), output.end(), [](T v) {
+    return v == T{2};
+  }));
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_for_cleanup(&build));
+}
+#endif // CCCL_C_PARALLEL_V2
+
+// TODO:
+/*
+C2H_TEST("for works with iterators", "[for]")
+{
+  const int num_items = GENERATE(1, 42, take(4, random(1 << 12, 1 << 16)));
+
+  iterator_t<int, constant_iterator_state_t<int>> input_it = make_iterator<int, constant_iterator_state_t<int>>(
+    {"constant_iterator_state_t", "struct constant_iterator_state_t { int value; };\n"},
+    {"in_advance", "extern \"C\" __device__ void in_advance(constant_iterator_state_t*, unsigned long long) {}"},
+    {"in_dereference",
+     "extern \"C\" __device__ void in_dereference(constant_iterator_state_t* state, int* result) { \n"
+     "  *result = state->value;\n"
+     "}"});
+  input_it.state.value = 1;
+
+  pointer_t<int> counter(1);
+  invocation_counter_state_t op_state                 = {counter.ptr};
+  stateful_operation_t<invocation_counter_state_t> op = make_operation(
+    "op",
+    R"XXX(
+struct invocation_counter_state_t { int* d_counter; };
+extern "C" __device__ void op(invocation_counter_state_t* state, int a) {
+  atomicAdd(state->d_counter, a);
+}
+)XXX",
+    op_state);
+
+  for_each_uncached(input_it, num_items, op);
+
+  const int invocation_count = counter[0];
+  REQUIRE(invocation_count == num_items);
+}
+*/

cccl_upstream/c/parallel/test/test_histogram.cpp

@@ -0,0 +1,540 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <array>
+#include <cstdint>
+#include <vector>
+
+#include <cuda_runtime.h>
+
+#include "algorithm_execution.h"
+#include "test_util.h"
+#include <cccl/c/histogram.h>
+
+using sample_types =
+  c2h::type_list<std::int8_t,
+                 std::uint16_t,
+                 std::int32_t,
+                 std::uint64_t,
+#if _CCCL_HAS_NVFP16()
+                 __half,
+#endif
+                 float,
+                 double>;
+
+constexpr int num_channels        = 1;
+constexpr int num_active_channels = 1;
+
+void build_histogram(
+  cccl_device_histogram_build_result_t* build,
+  cccl_iterator_t d_samples,
+  int num_output_levels_val,
+  cccl_iterator_t d_output_histograms,
+  cccl_type_info level_type,
+  uint64_t num_rows,
+  uint64_t row_stride_samples,
+  bool is_evenly_segmented)
+{
+  cudaDeviceProp deviceProp;
+  cudaGetDeviceProperties(&deviceProp, 0);
+
+  const int cc_major = deviceProp.major;
+  const int cc_minor = deviceProp.minor;
+
+  const char* cub_path        = TEST_CUB_PATH;
+  const char* thrust_path     = TEST_THRUST_PATH;
+  const char* libcudacxx_path = TEST_LIBCUDACXX_PATH;
+  const char* ctk_path        = TEST_CTK_PATH;
+
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_histogram_build(
+      build,
+      num_channels,
+      num_active_channels,
+      d_samples,
+      num_output_levels_val,
+      d_output_histograms,
+      level_type,
+      num_rows,
+      row_stride_samples,
+      is_evenly_segmented,
+      cc_major,
+      cc_minor,
+      cub_path,
+      thrust_path,
+      libcudacxx_path,
+      ctk_path));
+}
+
+void histogram_even(
+  cccl_iterator_t d_samples,
+  cccl_iterator_t d_output_histograms,
+  cccl_value_t num_output_levels,
+  int num_output_levels_val,
+  cccl_value_t lower_level,
+  cccl_value_t upper_level,
+  int64_t num_row_pixels,
+  int64_t num_rows,
+  int64_t row_stride_samples)
+{
+  cccl_device_histogram_build_result_t build{};
+  build_histogram(
+    &build, d_samples, num_output_levels_val, d_output_histograms, lower_level.type, num_rows, row_stride_samples, true);
+
+  size_t temp_storage_bytes = 0;
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_histogram_even(
+      build,
+      nullptr,
+      &temp_storage_bytes,
+      d_samples,
+      d_output_histograms,
+      num_output_levels,
+      lower_level,
+      upper_level,
+      num_row_pixels,
+      num_rows,
+      row_stride_samples,
+      nullptr));
+
+  pointer_t<uint8_t> temp_storage(temp_storage_bytes);
+
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_histogram_even(
+      build,
+      temp_storage.ptr,
+      &temp_storage_bytes,
+      d_samples,
+      d_output_histograms,
+      num_output_levels,
+      lower_level,
+      upper_level,
+      num_row_pixels,
+      num_rows,
+      row_stride_samples,
+      nullptr));
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_histogram_cleanup(&build));
+}
+
+// Copied from catch2_test_device_histogram.cu (With some modifications)
+template <size_t ActiveChannels>
+auto generate_level_counts_to_test(int max_level_count) -> std::vector<int>
+{
+  // first channel tests maximum number of levels, later channels less and less
+  std::vector<int> r{max_level_count};
+  for (size_t c = 1; c < ActiveChannels; ++c)
+  {
+    r[c] = r[c - 1] / 2 + 1;
+  }
+  return r;
+}
+
+template <size_t ActiveChannels, typename LevelT>
+auto setup_bin_levels_for_even(const std::vector<int>& num_levels, LevelT max_level, int max_level_count)
+  -> std::vector<std::vector<LevelT>>
+{
+  std::vector<std::vector<LevelT>> levels(2);
+  auto& lower_level = levels[0];
+  auto& upper_level = levels[1];
+
+  lower_level.resize(ActiveChannels);
+  upper_level.resize(ActiveChannels);
+
+  // Create upper and lower levels between between [0:max_level], getting narrower with each channel. Example:
+  //    max_level = 256
+  //   num_levels = { 257, 129,  65 }
+  //  lower_level = {   0,  64,  96 }
+  //  upper_level = { 256, 192, 160 }
+
+  const auto min_bin_width = max_level / (max_level_count - 1);
+  REQUIRE(min_bin_width > 0);
+
+  for (size_t c = 0; c < ActiveChannels; ++c)
+  {
+    const int num_bins        = num_levels[c] - 1;
+    const auto min_hist_width = num_bins * min_bin_width;
+    lower_level[c]            = static_cast<LevelT>(max_level / 2 - min_hist_width / 2);
+    upper_level[c]            = static_cast<LevelT>(max_level / 2 + min_hist_width / 2);
+    REQUIRE(lower_level[c] < upper_level[c]);
+  }
+  return levels;
+}
+
+template <int Channels, typename counter_t, size_t ActiveChannels, typename SampleT, typename TransformOp, typename OffsetT>
+auto compute_reference_result(
+  const std::vector<SampleT>& h_samples,
+  const TransformOp& sample_to_bin_index,
+  const std::vector<int>& num_levels,
+  OffsetT width,
+  OffsetT height,
+  OffsetT row_pitch) -> std::array<std::vector<counter_t>, ActiveChannels>
+{
+  auto h_histogram = std::array<std::vector<counter_t>, ActiveChannels>{};
+  for (size_t c = 0; c < ActiveChannels; ++c)
+  {
+    h_histogram[c].resize(num_levels[c] - 1);
+  }
+  for (OffsetT row = 0; row < height; ++row)
+  {
+    for (OffsetT pixel = 0; pixel < width; ++pixel)
+    {
+      for (size_t c = 0; c < ActiveChannels; ++c)
+      {
+        const auto offset = row * (row_pitch / sizeof(SampleT)) + pixel * Channels + c;
+        const int bin     = sample_to_bin_index(static_cast<int>(c), h_samples[offset]);
+        if (bin >= 0 && bin < static_cast<int>(h_histogram[c].size())) // if bin is valid
+        {
+          ++h_histogram[c][bin];
+        }
+      }
+    }
+  }
+  return h_histogram;
+}
+
+C2H_TEST("DeviceHistogram::HistogramEven API usage", "[histogram][device]")
+{
+  using counter_t = int;
+  using level_t   = float;
+
+  int num_samples = 10;
+  std::vector<float> d_samples{2.2f, 6.1f, 7.1f, 2.9f, 3.5f, 0.3f, 2.9f, 2.1f, 6.1f, 999.5f};
+
+  int num_rows = 1;
+
+  int num_levels = 7;
+  std::vector<int> d_num_levels{num_levels};
+  std::vector<counter_t> d_single_histogram(6, 0);
+  pointer_t<counter_t> d_single_histogram_ptr(d_single_histogram);
+
+  level_t lower_level = 0.0;
+  level_t upper_level = 12.0;
+
+  pointer_t<float> d_samples_ptr(d_samples);
+  value_t<int> num_levels_val{num_levels};
+  pointer_t<int> d_num_levels_ptr(d_num_levels);
+
+  value_t<level_t> lower_level_val{lower_level};
+  value_t<level_t> upper_level_val{upper_level};
+
+  int64_t row_stride_samples = static_cast<int64_t>(num_samples);
+
+  histogram_even(
+    d_samples_ptr,
+    d_single_histogram_ptr,
+    num_levels_val,
+    num_levels,
+    lower_level_val,
+    upper_level_val,
+    num_samples,
+    num_rows,
+    row_stride_samples);
+
+  std::vector<counter_t> d_histogram_out(d_single_histogram_ptr);
+  CHECK(d_histogram_out == std::vector{1, 5, 0, 3, 0, 0});
+}
+
+C2H_TEST("DeviceHistogram::HistogramEven basic use", "[histogram][device]", sample_types)
+{
+  using counter_t = int;
+  using sample_t  = c2h::get<0, TestType>;
+  using offset_t  = int;
+  using level_t   = std::conditional_t<std::is_floating_point_v<sample_t>, sample_t, int>;
+
+  const auto max_level       = level_t{sizeof(sample_t) == 1 ? 126 : 1024};
+  const auto max_level_count = (sizeof(sample_t) == 1 ? 126 : 1024) + 1;
+
+  offset_t width  = 1920;
+  offset_t height = 1080;
+
+  constexpr int channels        = 1;
+  constexpr int active_channels = 1;
+
+  const auto padding_bytes     = static_cast<offset_t>(GENERATE(size_t{0}, 13 * sizeof(sample_t)));
+  const offset_t row_pitch     = width * channels * sizeof(sample_t) + padding_bytes;
+  const auto num_levels        = generate_level_counts_to_test<active_channels>(max_level_count);
+  const offset_t total_samples = height * (row_pitch / sizeof(sample_t));
+
+  std::vector<int64_t> samples_gen = generate<int64_t>(total_samples);
+  std::vector<sample_t> h_samples(total_samples);
+  for (int i = 0; i < total_samples; i++)
+  {
+    h_samples[i] = static_cast<sample_t>(samples_gen[i]);
+  }
+
+  std::vector<counter_t> d_single_histogram(num_levels[0] - 1, 0);
+
+  auto levels = setup_bin_levels_for_even<active_channels, level_t>(num_levels, max_level, max_level_count);
+
+  auto& lower_level = levels[0];
+  auto& upper_level = levels[1];
+
+  // Compute reference result
+  auto fp_scales = ::cuda::std::array<level_t, active_channels>{}; // only used when LevelT is floating point
+  for (size_t c = 0; c < active_channels; ++c)
+  {
+    if constexpr (!std::is_integral_v<level_t>)
+    {
+      fp_scales[c] = static_cast<level_t>(num_levels[c] - 1) / static_cast<level_t>(upper_level[c] - lower_level[c]);
+    }
+  }
+
+  auto sample_to_bin_index = [&](int channel, sample_t sample) {
+    using common_t             = ::cuda::std::common_type_t<level_t, sample_t>;
+    const auto n               = num_levels[channel];
+    const auto max             = static_cast<common_t>(upper_level[channel]);
+    const auto min             = static_cast<common_t>(lower_level[channel]);
+    const auto promoted_sample = static_cast<common_t>(sample);
+    if (promoted_sample < min || promoted_sample >= max)
+    {
+      return n; // out of range
+    }
+    if constexpr (::cuda::std::is_integral<level_t>::value)
+    {
+      // Accurate bin computation following the arithmetic we guarantee in the HistoEven docs
+      return static_cast<int>(
+        static_cast<uint64_t>(promoted_sample - min) * static_cast<uint64_t>(n - 1) / static_cast<uint64_t>(max - min));
+    }
+    else
+    {
+      return static_cast<int>((static_cast<common_t>(sample) - min) * fp_scales[channel]);
+    }
+    _CCCL_UNREACHABLE();
+  };
+  auto h_histogram = compute_reference_result<channels, counter_t, active_channels>(
+    h_samples, sample_to_bin_index, num_levels, width, height, row_pitch);
+
+  // Compute result and verify
+  pointer_t<sample_t> sample_ptr(h_samples);
+  pointer_t<counter_t> d_single_histogram_ptr(d_single_histogram);
+
+  value_t<int> num_levels_val{num_levels[0]};
+  value_t<level_t> lower_level_val{lower_level[0]};
+  value_t<level_t> upper_level_val{upper_level[0]};
+
+  histogram_even(
+    sample_ptr,
+    d_single_histogram_ptr,
+    num_levels_val,
+    num_levels[0],
+    lower_level_val,
+    upper_level_val,
+    width,
+    height,
+    row_pitch / sizeof(sample_t));
+
+  for (size_t c = 0; c < active_channels; ++c)
+  {
+    CHECK(h_histogram[c] == std::vector<counter_t>(d_single_histogram_ptr));
+  }
+}
+
+C2H_TEST("DeviceHistogram::HistogramEven sample iterator", "[histogram][device]")
+{
+  using counter_t = int;
+  using sample_t  = std::int32_t;
+  using offset_t  = int;
+  using level_t   = int;
+
+  const auto max_level_count = 1025;
+
+  const auto num_levels = generate_level_counts_to_test<num_active_channels>(max_level_count);
+  const int num_bins    = num_levels[0] - 1;
+
+  const offset_t samples_per_bin        = 10;
+  const offset_t adjusted_total_samples = num_bins * samples_per_bin;
+
+  // Set up iterator that counts from 0 to adjusted_total_samples - 1
+  iterator_t<sample_t, counting_iterator_state_t<sample_t>> counting_it = make_counting_iterator<sample_t>("int");
+  counting_it.state.value                                               = static_cast<sample_t>(0);
+
+  std::vector<counter_t> d_single_histogram(num_levels[0] - 1, 0);
+
+  // Set up levels so that values 0 to adjusted_total_samples-1 are evenly distributed
+  std::vector<std::vector<level_t>> levels(2);
+  auto& lower_level = levels[0];
+  auto& upper_level = levels[1];
+
+  lower_level.resize(num_active_channels);
+  upper_level.resize(num_active_channels);
+
+  lower_level[0] = static_cast<level_t>(0);
+  upper_level[0] = static_cast<level_t>(adjusted_total_samples);
+
+  // Compute reference result - each bin should have exactly samples_per_bin elements
+  auto h_histogram = std::array<std::vector<counter_t>, num_active_channels>{};
+  h_histogram[0].resize(num_levels[0] - 1, samples_per_bin);
+
+  // Compute result and verify
+  pointer_t<counter_t> d_single_histogram_ptr(d_single_histogram);
+
+  value_t<int> num_levels_val{num_levels[0]};
+  value_t<level_t> lower_level_val{lower_level[0]};
+  value_t<level_t> upper_level_val{upper_level[0]};
+
+  histogram_even(
+    counting_it,
+    d_single_histogram_ptr,
+    num_levels_val,
+    num_levels[0],
+    lower_level_val,
+    upper_level_val,
+    adjusted_total_samples,
+    1,
+    adjusted_total_samples);
+
+  for (size_t c = 0; c < num_active_channels; ++c)
+  {
+    CHECK(h_histogram[c] == std::vector<counter_t>(d_single_histogram_ptr));
+  }
+}
+
+#ifndef CCCL_C_PARALLEL_V2
+C2H_TEST("Histogram build result has serialization metadata populated", "[histogram][device][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  pointer_t<T> samples(1);
+  pointer_t<T> histograms(1);
+  cccl_device_histogram_build_result_t build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_histogram_build(
+      &build,
+      /*num_channels=*/1,
+      /*num_active_channels=*/1,
+      samples,
+      /*num_output_levels_val=*/3,
+      histograms,
+      get_type_info<T>(),
+      /*num_rows=*/1,
+      /*row_stride_samples=*/1,
+      /*is_evenly_segmented=*/true,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path()));
+
+  CHECK(build.cc == build_info.get_cc_major() * 10 + build_info.get_cc_minor());
+  CHECK((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  CHECK(build.payload_size > 0);
+  CHECK(build.runtime_policy != nullptr);
+  CHECK(build.runtime_policy_size > 0);
+  REQUIRE(build.init_kernel_lowered_name != nullptr);
+  CHECK(build.init_kernel_lowered_name[0] != '\0');
+  REQUIRE(build.sweep_kernel_lowered_name != nullptr);
+  CHECK(build.sweep_kernel_lowered_name[0] != '\0');
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_histogram_cleanup(&build));
+}
+
+C2H_TEST("Histogram compile/load round-trip", "[histogram][device][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  pointer_t<T> dummy_samples(1);
+  pointer_t<T> dummy_histograms(1);
+  value_t<T> lower_level_val{T{0}};
+
+  cccl_device_histogram_build_result_t build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_histogram_compile(
+      &build,
+      /*num_channels=*/1,
+      /*num_active_channels=*/1,
+      dummy_samples,
+      /*num_output_levels_val=*/3,
+      dummy_histograms,
+      get_type_info<T>(),
+      /*num_rows=*/1,
+      /*row_stride_samples=*/1,
+      /*is_evenly_segmented=*/true,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+
+  REQUIRE((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  REQUIRE(build.payload_size > 0);
+  REQUIRE(build.init_kernel_lowered_name != nullptr);
+  REQUIRE(build.sweep_kernel_lowered_name != nullptr);
+  CHECK(build.library == nullptr);
+  CHECK(build.init_kernel == nullptr);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_histogram_load(&build));
+  REQUIRE(build.library != nullptr);
+  CHECK(build.init_kernel != nullptr);
+  CHECK(build.sweep_kernel != nullptr);
+
+  // 16 samples uniformly in [0, 4), 2 bins: [0,2) and [2,4)
+  constexpr std::size_t n_samples = 16;
+  const std::vector<T> samples    = {0, 0, 1, 1, 2, 2, 3, 3, 0, 1, 2, 3, 0, 1, 2, 3};
+  pointer_t<T> samples_ptr(samples);
+  pointer_t<T> histogram_ptr(2); // 2 bins
+  value_t<T> upper_level_val{T{4}};
+  value_t<int> num_levels_val{3}; // 3 level boundaries → 2 bins
+  CUstream null_stream      = nullptr;
+  size_t temp_storage_bytes = 0;
+
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_histogram_even(
+      build,
+      nullptr,
+      &temp_storage_bytes,
+      samples_ptr,
+      histogram_ptr,
+      num_levels_val,
+      lower_level_val,
+      upper_level_val,
+      /*num_row_pixels=*/static_cast<int64_t>(n_samples),
+      /*num_rows=*/1,
+      /*row_stride_samples=*/static_cast<int64_t>(n_samples),
+      null_stream));
+  pointer_t<uint8_t> temp_storage(temp_storage_bytes);
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_histogram_even(
+      build,
+      temp_storage.ptr,
+      &temp_storage_bytes,
+      samples_ptr,
+      histogram_ptr,
+      num_levels_val,
+      lower_level_val,
+      upper_level_val,
+      /*num_row_pixels=*/static_cast<int64_t>(n_samples),
+      /*num_rows=*/1,
+      /*row_stride_samples=*/static_cast<int64_t>(n_samples),
+      null_stream));
+
+  // samples {0,0,1,1,0,1,0,1} in bin 0 (8 samples) and {2,2,3,3,2,3,2,3} in bin 1 (8 samples)
+  REQUIRE(histogram_ptr[0] == 8);
+  REQUIRE(histogram_ptr[1] == 8);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_histogram_cleanup(&build));
+}
+#endif // CCCL_C_PARALLEL_V2

cccl_upstream/c/parallel/test/test_identity.h

@@ -0,0 +1,19 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#ifdef TEST_IDENTITY_ENABLED
+template <typename T>
+__device__ T test_identity(T value)
+{
+  return value;
+}
+#endif

cccl_upstream/c/parallel/test/test_merge_sort.cpp

@@ -0,0 +1,820 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <cstdint>
+#include <iostream>
+#include <optional>
+#include <string>
+
+#include <cuda_runtime.h>
+
+#include "algorithm_execution.h"
+#include "build_result_caching.h"
+#include "test_util.h"
+#include <cccl/c/merge_sort.h>
+
+using key_types =
+  c2h::type_list<uint8_t,
+                 int16_t,
+                 uint32_t,
+#if _CCCL_HAS_NVFP16()
+                 __half,
+#endif
+                 double>;
+using item_t = float;
+
+using BuildResultT = cccl_device_merge_sort_build_result_t;
+
+struct merge_sort_cleanup
+{
+  CUresult operator()(BuildResultT* build_data) const noexcept
+  {
+    return cccl_device_merge_sort_cleanup(build_data);
+  }
+};
+
+using merge_sort_deleter       = BuildResultDeleter<BuildResultT, merge_sort_cleanup>;
+using merge_sort_build_cache_t = build_cache_t<std::string, result_wrapper_t<BuildResultT, merge_sort_deleter>>;
+
+template <typename Tag>
+auto& get_cache()
+{
+  return fixture<merge_sort_build_cache_t, Tag>::get_or_create().get_value();
+}
+
+template <bool DisableSassCheck = false>
+struct merge_sort_build
+{
+  template <typename... Rest>
+  CUresult operator()(
+    BuildResultT* build_ptr,
+    cccl_iterator_t input_keys,
+    cccl_iterator_t input_items,
+    cccl_iterator_t output_keys,
+    cccl_iterator_t output_items,
+    uint64_t,
+    cccl_op_t op,
+    Rest... rest) const noexcept
+  {
+    return cccl_device_merge_sort_build(build_ptr, input_keys, input_items, output_keys, output_items, op, rest...);
+  }
+
+  static constexpr bool should_check_sass(int)
+  {
+    return !DisableSassCheck;
+  }
+};
+
+struct merge_sort_run
+{
+  template <typename... Args>
+  CUresult operator()(Args... args) const noexcept
+  {
+    return cccl_device_merge_sort(args...);
+  }
+};
+
+template <bool DisableSassCheck = false, typename BuildCache = merge_sort_build_cache_t, typename KeyT = std::string>
+void merge_sort(
+  cccl_iterator_t input_keys,
+  cccl_iterator_t input_items,
+  cccl_iterator_t output_keys,
+  cccl_iterator_t output_items,
+  uint64_t num_items,
+  cccl_op_t op,
+  std::optional<BuildCache>& cache,
+  const std::optional<KeyT>& lookup_key)
+{
+  AlgorithmExecute<BuildResultT, merge_sort_build<DisableSassCheck>, merge_sort_cleanup, merge_sort_run, BuildCache, KeyT>(
+    cache, lookup_key, input_keys, input_items, output_keys, output_items, num_items, op);
+}
+
+// ================
+//   Start of tests
+// ================
+
+struct DeviceMergeSort_SortKeys_Fixture_Tag;
+C2H_TEST("DeviceMergeSort::SortKeys works", "[merge_sort]", key_types)
+{
+  using key_t = c2h::get<0, TestType>;
+
+  const int num_items = GENERATE_COPY(take(2, random(1, 1000000)), values({500, 1000000, 2000000}));
+
+  operation_t op                   = make_operation("op", get_merge_sort_op(get_type_info<key_t>().type));
+  std::vector<key_t> input_keys    = make_shuffled_sequence<key_t>(num_items);
+  std::vector<key_t> expected_keys = input_keys;
+
+  pointer_t<key_t> input_keys_it(input_keys);
+  pointer_t<key_t> input_items_it;
+
+  auto& build_cache    = get_cache<DeviceMergeSort_SortKeys_Fixture_Tag>();
+  const auto& test_key = make_key<key_t>();
+
+  merge_sort(input_keys_it, input_items_it, input_keys_it, input_items_it, num_items, op, build_cache, test_key);
+
+  std::sort(expected_keys.begin(), expected_keys.end());
+  REQUIRE(expected_keys == std::vector<key_t>(input_keys_it));
+}
+
+struct DeviceMergeSort_SortKeys_WellKnown_Fixture_Tag;
+C2H_TEST("DeviceMergeSort::SortKeys works with well-known predicate", "[merge_sort][well_known]", key_types)
+{
+  using key_t = c2h::get<0, TestType>;
+
+  const int num_items = GENERATE_COPY(take(2, random(1, 1000000)), values({500, 1000000, 2000000}));
+
+  cccl_op_t op                     = make_well_known_less_binary_predicate();
+  std::vector<key_t> input_keys    = make_shuffled_sequence<key_t>(num_items);
+  std::vector<key_t> expected_keys = input_keys;
+
+  pointer_t<key_t> input_keys_it(input_keys);
+  pointer_t<key_t> input_items_it;
+
+  auto& build_cache    = get_cache<DeviceMergeSort_SortKeys_WellKnown_Fixture_Tag>();
+  const auto& test_key = make_key<key_t>();
+
+  merge_sort(input_keys_it, input_items_it, input_keys_it, input_items_it, num_items, op, build_cache, test_key);
+
+  std::sort(expected_keys.begin(), expected_keys.end());
+  REQUIRE(expected_keys == std::vector<key_t>(input_keys_it));
+}
+
+struct DeviceMergeSort_SortKeysCopy_Fixture_Tag;
+C2H_TEST("DeviceMergeSort::SortKeysCopy works", "[merge_sort]", key_types)
+{
+  using key_t = c2h::get<0, TestType>;
+
+  const int num_items = GENERATE_COPY(take(2, random(1, 1000000)), values({500, 1000000, 2000000}));
+
+  operation_t op                = make_operation("op", get_merge_sort_op(get_type_info<key_t>().type));
+  std::vector<key_t> input_keys = make_shuffled_sequence<key_t>(num_items);
+  std::vector<key_t> output_keys(num_items);
+  std::vector<key_t> expected_keys = input_keys;
+
+  pointer_t<key_t> input_keys_it(input_keys);
+  pointer_t<key_t> input_items_it;
+  pointer_t<key_t> output_keys_it(output_keys);
+
+  auto& build_cache    = get_cache<DeviceMergeSort_SortKeysCopy_Fixture_Tag>();
+  const auto& test_key = make_key<key_t>();
+
+  merge_sort(input_keys_it, input_items_it, output_keys_it, input_items_it, num_items, op, build_cache, test_key);
+
+  std::sort(expected_keys.begin(), expected_keys.end());
+  REQUIRE(expected_keys == std::vector<key_t>(output_keys_it));
+}
+
+struct DeviceMergeSort_SortPairs_Fixture_Tag;
+C2H_TEST("DeviceMergeSort::SortPairs works", "[merge_sort]", key_types)
+{
+  using key_t = c2h::get<0, TestType>;
+
+  const int num_items = GENERATE_COPY(take(2, random(1, 1000000)), values({500, 1000000, 2000000}));
+
+  operation_t op                = make_operation("op", get_merge_sort_op(get_type_info<key_t>().type));
+  std::vector<key_t> input_keys = make_shuffled_sequence<key_t>(num_items);
+  std::vector<item_t> input_items(num_items);
+  std::transform(input_keys.begin(), input_keys.end(), input_items.begin(), [](key_t key) {
+    return static_cast<item_t>(key);
+  });
+  std::vector<key_t> expected_keys   = input_keys;
+  std::vector<item_t> expected_items = input_items;
+
+  pointer_t<key_t> input_keys_it(input_keys);
+  pointer_t<item_t> input_items_it(input_items);
+
+  auto& build_cache    = get_cache<DeviceMergeSort_SortPairs_Fixture_Tag>();
+  const auto& test_key = make_key<key_t, item_t>();
+
+  merge_sort<true>(input_keys_it, input_items_it, input_keys_it, input_items_it, num_items, op, build_cache, test_key);
+
+  std::sort(expected_keys.begin(), expected_keys.end());
+  std::sort(expected_items.begin(), expected_items.end());
+  REQUIRE(expected_keys == std::vector<key_t>(input_keys_it));
+  REQUIRE(expected_items == std::vector<item_t>(input_items_it));
+}
+
+struct DeviceMergeSort_SortPairsCopy_Fixture_Tag;
+C2H_TEST("DeviceMergeSort::SortPairsCopy works ", "[merge_sort]", key_types)
+{
+  using key_t = c2h::get<0, TestType>;
+
+  const int num_items = GENERATE_COPY(take(2, random(1, 1000000)), values({500, 1000000, 2000000}));
+
+  operation_t op                = make_operation("op", get_merge_sort_op(get_type_info<key_t>().type));
+  std::vector<key_t> input_keys = make_shuffled_sequence<key_t>(num_items);
+  std::vector<item_t> input_items(num_items);
+  std::transform(input_keys.begin(), input_keys.end(), input_items.begin(), [](key_t key) {
+    return static_cast<item_t>(key);
+  });
+  std::vector<key_t> output_keys(num_items);
+  std::vector<item_t> output_items(num_items);
+  std::vector<key_t> expected_keys   = input_keys;
+  std::vector<item_t> expected_items = input_items;
+
+  pointer_t<key_t> input_keys_it(input_keys);
+  pointer_t<item_t> input_items_it(input_items);
+  pointer_t<key_t> output_keys_it(output_keys);
+  pointer_t<item_t> output_items_it(output_items);
+
+  auto& build_cache    = get_cache<DeviceMergeSort_SortPairs_Fixture_Tag>();
+  const auto& test_key = make_key<key_t, item_t>();
+
+  merge_sort<true>(input_keys_it, input_items_it, output_keys_it, output_items_it, num_items, op, build_cache, test_key);
+
+  std::sort(expected_keys.begin(), expected_keys.end());
+  std::sort(expected_items.begin(), expected_items.end());
+  REQUIRE(expected_keys == std::vector<key_t>(output_keys_it));
+  REQUIRE(expected_items == std::vector<item_t>(output_items_it));
+}
+
+struct key_pair
+{
+  short a;
+  size_t b;
+};
+
+struct item_pair
+{
+  int a;
+  float b;
+};
+
+struct DeviceMergeSort_SortPairsCopy_CustomType_Fixture_Tag;
+C2H_TEST("DeviceMergeSort:SortPairsCopy works with custom types", "[merge_sort]")
+{
+  const size_t num_items      = GENERATE_COPY(take(2, random(1, 100000)), values({5, 10000, 100000}));
+  operation_t op              = make_operation("op",
+                                  R"(struct key_pair { short a; size_t b; };
+extern "C" __device__ void op(void* lhs_ptr, void* rhs_ptr, bool* out_ptr) {
+  key_pair* lhs = static_cast<key_pair*>(lhs_ptr);
+  key_pair* rhs = static_cast<key_pair*>(rhs_ptr);
+  bool* out = static_cast<bool*>(out_ptr);
+  *out = lhs->a == rhs->a ? lhs->b < rhs->b : lhs->a < rhs->a;
+})");
+  const std::vector<short> a  = generate<short>(num_items);
+  const std::vector<size_t> b = generate<size_t>(num_items);
+  std::vector<key_pair> input_keys(num_items);
+  std::vector<item_pair> input_items(num_items);
+  for (std::size_t i = 0; i < num_items; ++i)
+  {
+    input_keys[i]  = key_pair{a[i], b[i]};
+    input_items[i] = item_pair{static_cast<int>(a[i]), static_cast<float>(b[i])};
+  }
+  std::vector<key_pair> expected_keys   = input_keys;
+  std::vector<item_pair> expected_items = input_items;
+
+  pointer_t<key_pair> input_keys_it(input_keys);
+  pointer_t<item_pair> input_items_it(input_items);
+  pointer_t<key_pair> output_keys_it(input_keys);
+  pointer_t<item_pair> output_items_it(input_items);
+
+  auto& build_cache    = get_cache<DeviceMergeSort_SortPairsCopy_CustomType_Fixture_Tag>();
+  const auto& test_key = make_key<key_pair, item_pair>();
+
+  merge_sort(input_keys_it, input_items_it, output_keys_it, output_items_it, num_items, op, build_cache, test_key);
+
+  std::sort(expected_keys.begin(), expected_keys.end(), [](const key_pair& lhs, const key_pair& rhs) {
+    return lhs.a == rhs.a ? lhs.b < rhs.b : lhs.a < rhs.a;
+  });
+  std::sort(expected_items.begin(), expected_items.end(), [](const item_pair& lhs, const item_pair& rhs) {
+    return lhs.a == rhs.a ? lhs.b < rhs.b : lhs.a < rhs.a;
+  });
+  REQUIRE(std::equal(
+    expected_keys.begin(),
+    expected_keys.end(),
+    std::vector<key_pair>(output_keys_it).begin(),
+    [](const key_pair& lhs, const key_pair& rhs) {
+      return lhs.a == rhs.a && lhs.b == rhs.b;
+    }));
+  REQUIRE(std::equal(
+    expected_items.begin(),
+    expected_items.end(),
+    std::vector<item_pair>(output_items_it).begin(),
+    [](const item_pair& lhs, const item_pair& rhs) {
+      return lhs.a == rhs.a && lhs.b == rhs.b;
+    }));
+}
+
+struct DeviceMergeSort_SortPairsCopy_CustomType_WellKnown_Fixture_Tag;
+C2H_TEST("DeviceMergeSort:SortPairsCopy works with custom types with well-known predicates", "[merge_sort][well_known]")
+{
+  const size_t num_items      = GENERATE_COPY(take(2, random(1, 100000)), values({5, 10000, 100000}));
+  operation_t op_state        = make_operation("op",
+                                        R"(struct key_pair { short a; size_t b; };
+extern "C" __device__ void op(void* lhs_ptr, void* rhs_ptr, bool* out_ptr) {
+  key_pair* lhs = static_cast<key_pair*>(lhs_ptr);
+  key_pair* rhs = static_cast<key_pair*>(rhs_ptr);
+  bool* out = static_cast<bool*>(out_ptr);
+  *out = lhs->a == rhs->a ? lhs->b < rhs->b : lhs->a < rhs->a;
+})");
+  cccl_op_t op                = op_state;
+  op.type                     = cccl_op_kind_t::CCCL_LESS;
+  const std::vector<short> a  = generate<short>(num_items);
+  const std::vector<size_t> b = generate<size_t>(num_items);
+  std::vector<key_pair> input_keys(num_items);
+  std::vector<item_pair> input_items(num_items);
+  for (std::size_t i = 0; i < num_items; ++i)
+  {
+    input_keys[i]  = key_pair{a[i], b[i]};
+    input_items[i] = item_pair{static_cast<int>(a[i]), static_cast<float>(b[i])};
+  }
+  std::vector<key_pair> expected_keys   = input_keys;
+  std::vector<item_pair> expected_items = input_items;
+
+  pointer_t<key_pair> input_keys_it(input_keys);
+  pointer_t<item_pair> input_items_it(input_items);
+  pointer_t<key_pair> output_keys_it(input_keys);
+  pointer_t<item_pair> output_items_it(input_items);
+
+  auto& build_cache    = get_cache<DeviceMergeSort_SortPairsCopy_CustomType_WellKnown_Fixture_Tag>();
+  const auto& test_key = make_key<key_pair, item_pair>();
+
+  merge_sort(input_keys_it, input_items_it, output_keys_it, output_items_it, num_items, op, build_cache, test_key);
+
+  std::sort(expected_keys.begin(), expected_keys.end(), [](const key_pair& lhs, const key_pair& rhs) {
+    return lhs.a == rhs.a ? lhs.b < rhs.b : lhs.a < rhs.a;
+  });
+  std::sort(expected_items.begin(), expected_items.end(), [](const item_pair& lhs, const item_pair& rhs) {
+    return lhs.a == rhs.a ? lhs.b < rhs.b : lhs.a < rhs.a;
+  });
+  REQUIRE(std::equal(
+    expected_keys.begin(),
+    expected_keys.end(),
+    std::vector<key_pair>(output_keys_it).begin(),
+    [](const key_pair& lhs, const key_pair& rhs) {
+      return lhs.a == rhs.a && lhs.b == rhs.b;
+    }));
+  REQUIRE(std::equal(
+    expected_items.begin(),
+    expected_items.end(),
+    std::vector<item_pair>(output_items_it).begin(),
+    [](const item_pair& lhs, const item_pair& rhs) {
+      return lhs.a == rhs.a && lhs.b == rhs.b;
+    }));
+}
+
+struct DeviceMergeSort_SortKeys_Iterators_Fixture_Tag;
+C2H_TEST("DeviceMergeSort::SortKeys works with input iterators", "[merge_sort]")
+{
+  using T             = int;
+  const int num_items = GENERATE_COPY(take(2, random(1, 1000000)), values({500, 1000000, 2000000}));
+
+  operation_t op = make_operation("op", get_merge_sort_op(get_type_info<T>().type));
+  iterator_t<T, random_access_iterator_state_t<T>> input_keys_it =
+    make_random_access_iterator<T>(iterator_kind::INPUT, "int");
+  std::vector<T> input_keys    = make_shuffled_sequence<T>(num_items);
+  std::vector<T> expected_keys = input_keys;
+
+  pointer_t<T> input_keys_ptr(input_keys);
+  input_keys_it.state.data = input_keys_ptr.ptr;
+  pointer_t<T> input_items_it;
+
+  auto& build_cache    = get_cache<DeviceMergeSort_SortKeys_Iterators_Fixture_Tag>();
+  const auto& test_key = make_key<T>();
+
+  merge_sort(input_keys_it, input_items_it, input_keys_ptr, input_items_it, num_items, op, build_cache, test_key);
+
+  std::sort(expected_keys.begin(), expected_keys.end());
+  REQUIRE(expected_keys == std::vector<T>(input_keys_ptr));
+}
+
+struct DeviceMergeSort_SortPairs_Iterators_Fixture_Tag;
+C2H_TEST("DeviceMergeSort::SortPairs works with input iterators", "[merge_sort]")
+{
+  using key_t         = int;
+  using int_item_t    = int;
+  const int num_items = GENERATE_COPY(take(2, random(1, 1000000)), values({500, 1000000, 2000000}));
+
+  operation_t op = make_operation("op", get_merge_sort_op(get_type_info<key_t>().type));
+  iterator_t<key_t, random_access_iterator_state_t<key_t>> input_keys_it =
+    make_random_access_iterator<key_t>(iterator_kind::INPUT, "int", "key");
+  iterator_t<key_t, random_access_iterator_state_t<key_t>> input_items_it =
+    make_random_access_iterator<key_t>(iterator_kind::INPUT, "int", "item");
+
+  std::vector<key_t> input_keys = make_shuffled_sequence<key_t>(num_items);
+  std::vector<int_item_t> input_items(num_items);
+  std::transform(input_keys.begin(), input_keys.end(), input_items.begin(), [](key_t key) {
+    return static_cast<int_item_t>(key);
+  });
+
+  std::vector<key_t> expected_keys       = input_keys;
+  std::vector<int_item_t> expected_items = input_items;
+
+  pointer_t<key_t> input_keys_ptr(input_keys);
+  input_keys_it.state.data = input_keys_ptr.ptr;
+  pointer_t<key_t> input_items_ptr(input_items);
+  input_items_it.state.data = input_items_ptr.ptr;
+
+  auto& build_cache    = get_cache<DeviceMergeSort_SortPairs_Iterators_Fixture_Tag>();
+  const auto& test_key = make_key<key_t, int_item_t>();
+
+  merge_sort(input_keys_it, input_items_it, input_keys_ptr, input_items_ptr, num_items, op, build_cache, test_key);
+
+  std::sort(expected_keys.begin(), expected_keys.end());
+  std::sort(expected_items.begin(), expected_items.end());
+  REQUIRE(expected_keys == std::vector<key_t>(input_keys_ptr));
+  REQUIRE(expected_items == std::vector<int_item_t>(input_items_ptr));
+}
+
+// These tests with output iterators are currently failing https://github.com/NVIDIA/cccl/issues/3722
+#ifdef NEVER_DEFINED
+C2H_TEST("DeviceMergeSort::SortKeys works with output iterators", "[merge_sort]")
+{
+  using TestType      = int;
+  const int num_items = GENERATE_COPY(take(2, random(1, 1000000)), values({500, 1000000, 2000000}));
+
+  operation_t op = make_operation("op", get_merge_sort_op(get_type_info<TestType>().type));
+  iterator_t<TestType, random_access_iterator_state_t> output_keys_it =
+    make_iterator<TestType, random_access_iterator_state_t>(
+      {"random_access_iterator_state_t", "struct random_access_iterator_state_t { int* d_input; };\n"},
+      {"advance",
+       R"(extern "C" __device__ void advance(void* state, const void* offset) {
+  auto* typed_state = static_cast<random_access_iterator_state_t*>(state);
+  auto offset_val = *static_cast<const unsigned long long*>(offset);
+  typed_state->d_input += offset_val;
+})"},
+      {"dereference",
+       R"(extern "C" __device__ void dereference(void* state, const void* x) {
+  auto* typed_state = static_cast<random_access_iterator_state_t*>(state);
+  auto x_val = *static_cast<const int*>(x);
+  *typed_state->d_input = x_val;
+})"});
+  std::vector<TestType> input_keys    = make_shuffled_key_ranks_vector<TestType>(num_items);
+  std::vector<TestType> expected_keys = input_keys;
+
+  pointer_t<TestType> input_keys_it(input_keys);
+  pointer_t<TestType> input_items_it;
+  output_keys_it.state.d_input = input_keys_it.ptr;
+
+  merge_sort(input_keys_it, input_items_it, output_keys_it, input_items_it, num_items, op);
+
+  std::sort(expected_keys.begin(), expected_keys.end());
+  REQUIRE(expected_keys == std::vector<TestType>(input_keys_it));
+}
+
+C2H_TEST("DeviceMergeSort::SortPairs works with output iterators for items", "[merge_sort]")
+{
+  using TestType      = int;
+  using item_t        = int;
+  const int num_items = GENERATE_COPY(take(2, random(1, 1000000)), values({500, 1000000, 2000000}));
+
+  operation_t op                   = make_operation("op", get_merge_sort_op(get_type_info<TestType>().type));
+  std::vector<TestType> input_keys = make_shuffled_sequence<TestType>(num_items);
+  std::vector<item_t> input_items(num_items);
+  std::transform(input_keys.begin(), input_keys.end(), input_items.begin(), [](TestType key) {
+    return static_cast<item_t>(key);
+  });
+  std::vector<TestType> expected_keys = input_keys;
+  std::vector<item_t> expected_items  = input_items;
+
+  iterator_t<item_t, item_random_access_iterator_state_t> output_items_it =
+    make_iterator<TestType, item_random_access_iterator_state_t>(
+      "struct item_random_access_iterator_state_t { int* d_input; };\n",
+      {"advance",
+       R"(extern "C" __device__ void advance(void* state, const void* offset) {
+  auto* typed_state = static_cast<item_random_access_iterator_state_t*>(state);
+  auto offset_val = *static_cast<const unsigned long long*>(offset);
+  typed_state->d_input += offset_val;
+})"},
+      {"dereference",
+       R"(extern "C" __device__ void dereference(void* state, const void* x) {
+  auto* typed_state = static_cast<item_random_access_iterator_state_t*>(state);
+  auto x_val = *static_cast<const int*>(x);
+  *typed_state->d_input = x_val;
+})"});
+
+  pointer_t<TestType> input_keys_it(input_keys);
+  pointer_t<item_t> input_items_it(input_items);
+  output_items_it.state.d_input = input_items_it.ptr;
+
+  merge_sort(input_keys_it, input_items_it, input_keys_it, output_items_it, num_items, op);
+
+  std::sort(expected_keys.begin(), expected_keys.end());
+  std::sort(expected_items.begin(), expected_items.end());
+  REQUIRE(expected_keys == std::vector<TestType>(input_keys_it));
+  REQUIRE(expected_items == std::vector<item_t>(input_items_it));
+}
+
+#endif
+
+struct large_key_pair
+{
+  int a;
+  char c[100];
+};
+
+C2H_TEST("MergeSort works with C++ source operations", "[merge_sort]")
+{
+  using key_t = int32_t;
+
+  const std::size_t num_items = GENERATE(42, 1337, 42000);
+
+  // Create operation from C++ source instead of LTO-IR
+  std::string cpp_source = R"(
+    extern "C" __device__ void op(void* lhs, void* rhs, void* result) {
+      int* ilhs = (int*)lhs;
+      int* irhs = (int*)rhs;
+      bool* bresult = (bool*)result;
+      *bresult = *ilhs < *irhs;
+    }
+  )";
+
+  operation_t op = make_cpp_operation("op", cpp_source);
+
+  std::vector<key_t> input_keys = make_shuffled_sequence<key_t>(num_items);
+  pointer_t<key_t> input_keys_ptr(input_keys);
+  pointer_t<key_t> output_keys_ptr(num_items);
+
+  // Use int for items but won't actually use them
+  pointer_t<int> input_items_ptr;
+  pointer_t<int> output_items_ptr;
+
+  // Test key including flag that this uses C++ source
+  std::optional<std::string> test_key = std::format("cpp_source_test_{}_{}", num_items, typeid(key_t).name());
+
+  auto& cache = fixture<merge_sort_build_cache_t, DeviceMergeSort_SortKeys_Fixture_Tag>::get_or_create().get_value();
+  std::optional<merge_sort_build_cache_t> cache_opt = cache;
+
+  merge_sort(input_keys_ptr, input_items_ptr, output_keys_ptr, output_items_ptr, num_items, op, cache_opt, test_key);
+
+  const std::vector<key_t> output = output_keys_ptr;
+  std::vector<key_t> expected     = input_keys;
+  std::sort(expected.begin(), expected.end());
+  REQUIRE(output == expected);
+}
+
+C2H_TEST("MergeSort works with C++ source operations using custom headers", "[merge_sort]")
+{
+  using key_t = int32_t;
+
+  const std::size_t num_items = GENERATE(42, 1337, 42000);
+
+  // Create operation from C++ source that uses the identity function from header
+  std::string cpp_source = R"(
+    #include "test_identity.h"
+    extern "C" __device__ void op(void* lhs, void* rhs, void* result) {
+      int* ilhs = (int*)lhs;
+      int* irhs = (int*)rhs;
+      bool* bresult = (bool*)result;
+      int val_lhs = test_identity(*ilhs);
+      int val_rhs = test_identity(*irhs);
+      *bresult = val_lhs < val_rhs;
+    }
+  )";
+
+  operation_t op = make_cpp_operation("op", cpp_source);
+
+  std::vector<key_t> input_keys = make_shuffled_sequence<key_t>(num_items);
+  pointer_t<key_t> input_keys_ptr(input_keys);
+  pointer_t<key_t> output_keys_ptr(num_items);
+
+  // Use int for items but won't actually use them
+  pointer_t<int> input_items_ptr;
+  pointer_t<int> output_items_ptr;
+
+  // Test _ex version with custom build configuration
+  const char* extra_flags[] = {"-DTEST_IDENTITY_ENABLED"};
+  const char* extra_dirs[]  = {TEST_INCLUDE_PATH};
+  cccl_build_config config  = make_build_config(extra_flags, 1, extra_dirs, 1);
+
+  // Build with _ex version
+  cccl_device_merge_sort_build_result_t build{};
+  const auto& build_info = BuildInformation<>::init();
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_merge_sort_build_ex(
+      &build,
+      input_keys_ptr,
+      input_items_ptr,
+      output_keys_ptr,
+      output_items_ptr,
+      op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      &config));
+
+  // Execute the merge sort
+  void* d_temp_storage      = nullptr;
+  size_t temp_storage_bytes = 0;
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_merge_sort(
+      build,
+      d_temp_storage,
+      &temp_storage_bytes,
+      input_keys_ptr,
+      input_items_ptr,
+      output_keys_ptr,
+      output_items_ptr,
+      num_items,
+      op,
+      CU_STREAM_LEGACY));
+  pointer_t<char> temp_storage(temp_storage_bytes);
+  d_temp_storage = static_cast<void*>(temp_storage.ptr);
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_merge_sort(
+      build,
+      d_temp_storage,
+      &temp_storage_bytes,
+      input_keys_ptr,
+      input_items_ptr,
+      output_keys_ptr,
+      output_items_ptr,
+      num_items,
+      op,
+      CU_STREAM_LEGACY));
+
+  // Verify results
+  std::vector<key_t> output_keys(num_items);
+  cudaMemcpy(
+    output_keys.data(), static_cast<void*>(output_keys_ptr.ptr), sizeof(key_t) * num_items, cudaMemcpyDeviceToHost);
+  std::vector<key_t> expected_keys(num_items);
+  cudaMemcpy(
+    expected_keys.data(), static_cast<void*>(input_keys_ptr.ptr), sizeof(key_t) * num_items, cudaMemcpyDeviceToHost);
+  std::sort(expected_keys.begin(), expected_keys.end());
+  std::sort(expected_keys.begin(), expected_keys.end());
+  REQUIRE(output_keys == expected_keys);
+
+  // Cleanup
+  REQUIRE(CUDA_SUCCESS == cccl_device_merge_sort_cleanup(&build));
+}
+
+// TODO: We no longer fail to build for large types due to no vsmem. Instead, the build passes,
+// but we get a ptxas error about the kernel using too much shared memory.
+/* C2H_TEST("DeviceMergeSort:SortPairsCopy fails to build for large types due to no vsmem", "[merge_sort]")
+{
+  const size_t num_items = 1;
+  operation_t op = make_operation(
+    "op",
+    R"(struct large_key_pair { int a; char c[100]; };
+extern "C" __device__ bool op(large_key_pair lhs, large_key_pair rhs) {
+  return lhs.a < rhs.a;
+})");
+  const std::vector<int> a = generate<int>(num_items);
+  std::vector<large_key_pair> input_keys(num_items);
+  for (std::size_t i = 0; i < num_items; ++i)
+  {
+    input_keys[i] = large_key_pair{a[i], {}};
+  }
+
+  pointer_t<large_key_pair> input_keys_it(input_keys);
+  pointer_t<int> input_items_it;
+
+  cudaDeviceProp deviceProp;
+  cudaGetDeviceProperties(&deviceProp, 0);
+
+  const int cc_major = deviceProp.major;
+  const int cc_minor = deviceProp.minor;
+
+  const char* cub_path        = TEST_CUB_PATH;
+  const char* thrust_path     = TEST_THRUST_PATH;
+  const char* libcudacxx_path = TEST_LIBCUDACXX_PATH;
+  const char* ctk_path        = TEST_CTK_PATH;
+
+  cccl_device_merge_sort_build_result_t build{};
+  REQUIRE(
+    CUDA_ERROR_UNKNOWN
+    == cccl_device_merge_sort_build(
+      &build,
+      input_keys_it,
+      input_items_it,
+      input_keys_it,
+      input_items_it,
+      op,
+      cc_major,
+      cc_minor,
+      cub_path,
+      thrust_path,
+      libcudacxx_path,
+      ctk_path));
+}
+ */
+
+#ifndef CCCL_C_PARALLEL_V2
+C2H_TEST("MergeSort build result has serialization metadata populated", "[merge_sort][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  cccl_op_t op = make_well_known_binary_operation();
+  pointer_t<T> keys_in(1);
+  pointer_t<T> items_in(1);
+  pointer_t<T> keys_out(1);
+  pointer_t<T> items_out(1);
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_merge_sort_build(
+      &build,
+      keys_in,
+      items_in,
+      keys_out,
+      items_out,
+      op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path()));
+
+  CHECK(build.cc == build_info.get_cc_major() * 10 + build_info.get_cc_minor());
+  CHECK((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  CHECK(build.payload_size > 0);
+  CHECK(build.runtime_policy != nullptr);
+  CHECK(build.runtime_policy_size > 0);
+  REQUIRE(build.block_sort_kernel_lowered_name != nullptr);
+  CHECK(build.block_sort_kernel_lowered_name[0] != '\0');
+  REQUIRE(build.partition_kernel_lowered_name != nullptr);
+  CHECK(build.partition_kernel_lowered_name[0] != '\0');
+  REQUIRE(build.merge_kernel_lowered_name != nullptr);
+  CHECK(build.merge_kernel_lowered_name[0] != '\0');
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_merge_sort_cleanup(&build));
+}
+
+C2H_TEST("MergeSort compile/load round-trip", "[merge_sort][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  cccl_op_t op = make_well_known_less_binary_predicate();
+  pointer_t<T> dummy_keys_in(1);
+  pointer_t<T> dummy_items_in(1);
+  pointer_t<T> dummy_keys_out(1);
+  pointer_t<T> dummy_items_out(1);
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_merge_sort_compile(
+      &build,
+      dummy_keys_in,
+      dummy_items_in,
+      dummy_keys_out,
+      dummy_items_out,
+      op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+
+  REQUIRE((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  REQUIRE(build.payload_size > 0);
+  REQUIRE(build.block_sort_kernel_lowered_name != nullptr);
+  REQUIRE(build.partition_kernel_lowered_name != nullptr);
+  REQUIRE(build.merge_kernel_lowered_name != nullptr);
+  CHECK(build.library == nullptr);
+  CHECK(build.block_sort_kernel == nullptr);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_merge_sort_load(&build));
+  REQUIRE(build.library != nullptr);
+  CHECK(build.block_sort_kernel != nullptr);
+  CHECK(build.partition_kernel != nullptr);
+  CHECK(build.merge_kernel != nullptr);
+
+  constexpr std::size_t n    = 16;
+  const std::vector<T> input = generate<T>(n);
+  pointer_t<T> keys_in(input);
+  pointer_t<T> items_in(input);
+  pointer_t<T> keys_out(n);
+  pointer_t<T> items_out(n);
+  CUstream null_stream      = nullptr;
+  size_t temp_storage_bytes = 0;
+
+  REQUIRE(CUDA_SUCCESS
+          == cccl_device_merge_sort(
+            build, nullptr, &temp_storage_bytes, keys_in, items_in, keys_out, items_out, n, op, null_stream));
+  pointer_t<uint8_t> temp_storage(temp_storage_bytes);
+  REQUIRE(CUDA_SUCCESS
+          == cccl_device_merge_sort(
+            build, temp_storage.ptr, &temp_storage_bytes, keys_in, items_in, keys_out, items_out, n, op, null_stream));
+
+  std::vector<T> expected(input);
+  std::sort(expected.begin(), expected.end());
+  REQUIRE(expected == std::vector<T>(keys_out));
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_merge_sort_cleanup(&build));
+}
+#endif // CCCL_C_PARALLEL_V2

cccl_upstream/c/parallel/test/test_radix_sort.cpp

@@ -0,0 +1,471 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <cstdint>
+#include <optional> // std::optional
+#include <string>
+
+#include <cuda_runtime.h>
+
+#include "algorithm_execution.h"
+#include "build_result_caching.h"
+#include "test_util.h"
+#include <cccl/c/radix_sort.h>
+
+using key_types =
+  c2h::type_list<uint8_t,
+                 int16_t,
+                 uint32_t,
+#if _CCCL_HAS_NVFP16()
+                 __half,
+#endif
+                 double>;
+using item_t = float;
+
+template <typename KeyTy, typename ItemTy, bool descending = false, bool overwrite_okay = false>
+struct TestParameters
+{
+  using KeyT                             = KeyTy;
+  using ItemT                            = ItemTy;
+  static constexpr bool m_descending     = descending;
+  static constexpr bool m_overwrite_okay = overwrite_okay;
+
+  constexpr TestParameters() = default;
+
+  bool is_descending() const
+  {
+    return m_descending;
+  }
+  bool is_overwrite_okay() const
+  {
+    return m_overwrite_okay;
+  }
+};
+
+using test_params_tuple =
+  c2h::type_list<TestParameters<c2h::get<0, key_types>, item_t, false, false>,
+                 TestParameters<c2h::get<1, key_types>, item_t, true, false>,
+                 TestParameters<c2h::get<2, key_types>, item_t, false, true>,
+                 TestParameters<c2h::get<3, key_types>, item_t, true, true>>;
+
+using BuildResultT = cccl_device_radix_sort_build_result_t;
+
+struct radix_sort_cleanup
+{
+  CUresult operator()(BuildResultT* build_data) const noexcept
+  {
+    return cccl_device_radix_sort_cleanup(build_data);
+  }
+};
+
+using radix_sort_deleter       = BuildResultDeleter<BuildResultT, radix_sort_cleanup>;
+using radix_sort_build_cache_t = build_cache_t<std::string, result_wrapper_t<BuildResultT, radix_sort_deleter>>;
+
+template <typename Tag>
+auto& get_cache()
+{
+  return fixture<radix_sort_build_cache_t, Tag>::get_or_create().get_value();
+}
+
+template <bool CheckSASS = true>
+struct radix_sort_build
+{
+  static constexpr auto should_check_sass(int cc_major)
+  {
+    // TODO: re-enable w/ nvrtc version check
+    return CheckSASS && cc_major < 9;
+  }
+
+  // operator arguments are (build_ptr, <all_args_of_algo_driver>, cc_major, cc_minor, <paths>)
+  //   of all_args_of_algo_driver we pick out what gets passed to cccl_algo_build function
+  CUresult operator()(
+    BuildResultT* build_ptr,
+    cccl_sort_order_t sort_order,
+    cccl_iterator_t d_keys_in,
+    cccl_iterator_t,
+    cccl_iterator_t d_values_in,
+    cccl_iterator_t,
+    cccl_op_t decomposer,
+    const char* decomposer_return_type,
+    uint64_t,
+    int,
+    int,
+    bool,
+    int*,
+    int cc_major,
+    int cc_minor,
+    const char* cub_path,
+    const char* thrust_path,
+    const char* libcudacxx_path,
+    const char* ctk_path) const noexcept
+  {
+    return cccl_device_radix_sort_build(
+      build_ptr,
+      sort_order,
+      d_keys_in,
+      d_values_in,
+      decomposer,
+      decomposer_return_type,
+      cc_major,
+      cc_minor,
+      cub_path,
+      thrust_path,
+      libcudacxx_path,
+      ctk_path);
+  }
+};
+
+struct radix_sort_run
+{
+  template <typename... Rest>
+  CUresult operator()(
+    BuildResultT build,
+    void* temp_storage,
+    size_t* temp_storage_bytes,
+    cccl_sort_order_t,
+    cccl_iterator_t d_keys_in,
+    cccl_iterator_t d_keys_out,
+    cccl_iterator_t d_values_in,
+    cccl_iterator_t d_values_out,
+    cccl_op_t decomposer,
+    const char*,
+    Rest... rest) const noexcept
+  {
+    return cccl_device_radix_sort(
+      build, temp_storage, temp_storage_bytes, d_keys_in, d_keys_out, d_values_in, d_values_out, decomposer, rest...);
+  }
+};
+
+template <bool CheckSASS = true, typename BuildCache = radix_sort_build_cache_t, typename KeyT = std::string>
+void radix_sort(
+  cccl_sort_order_t sort_order,
+  cccl_iterator_t d_keys_in,
+  cccl_iterator_t d_keys_out,
+  cccl_iterator_t d_values_in,
+  cccl_iterator_t d_values_out,
+  cccl_op_t decomposer,
+  const char* decomposer_return_type,
+  uint64_t num_items,
+  int begin_bit,
+  int end_bit,
+  bool is_overwrite_okay,
+  int* selector,
+  std::optional<BuildCache>& cache,
+  const std::optional<KeyT>& lookup_key)
+{
+  AlgorithmExecute<BuildResultT, radix_sort_build<CheckSASS>, radix_sort_cleanup, radix_sort_run, BuildCache, KeyT>(
+    cache,
+    lookup_key,
+    sort_order,
+    d_keys_in,
+    d_keys_out,
+    d_values_in,
+    d_values_out,
+    decomposer,
+    decomposer_return_type,
+    num_items,
+    begin_bit,
+    end_bit,
+    is_overwrite_okay,
+    selector);
+}
+
+struct DeviceRadixSort_SortKeys_Fixture_Tag;
+C2H_TEST("DeviceRadixSort::SortKeys works", "[radix_sort]", test_params_tuple)
+{
+  using T     = c2h::get<0, TestType>;
+  using KeyT  = typename T::KeyT;
+  using ItemT = typename T::ItemT;
+
+  constexpr auto this_test_params = T();
+  // We want a mix of small and large sizes because different implementations will be called
+  const int num_items = GENERATE_COPY(take(2, random(1, 1000000)), values({500, 1000000, 2000000}));
+  bool is_descending  = this_test_params.is_descending();
+  const auto order    = is_descending ? CCCL_DESCENDING : CCCL_ASCENDING;
+
+  const int begin_bit          = 0;
+  const int end_bit            = sizeof(KeyT) * 8;
+  const bool is_overwrite_okay = this_test_params.is_overwrite_okay();
+  int selector                 = -1;
+
+  static constexpr cccl_op_t decomposer_no_op{};
+  static constexpr const char* unused_decomposer_retty = "";
+
+  // problem descriptor: (order, TestType, item_t, is_overwrite_ok, items_present = false)
+  std::vector<KeyT> input_keys    = make_shuffled_sequence<KeyT>(num_items);
+  std::vector<KeyT> expected_keys = input_keys;
+
+  pointer_t<KeyT> input_keys_it(input_keys);
+  pointer_t<KeyT> output_keys_it(num_items);
+
+  pointer_t<ItemT> input_items_it, output_items_it;
+
+  auto& build_cache = get_cache<DeviceRadixSort_SortKeys_Fixture_Tag>();
+
+  const std::string& key_string = KeyBuilder::join(
+    {KeyBuilder::bool_as_key(is_descending),
+     KeyBuilder::type_as_key<T>(),
+     KeyBuilder::type_as_key<item_t>(),
+     KeyBuilder::bool_as_key(is_overwrite_okay)});
+  const auto& test_key = std::make_optional(key_string);
+
+  radix_sort(
+    order,
+    input_keys_it,
+    output_keys_it,
+    input_items_it,
+    output_items_it,
+    decomposer_no_op,
+    unused_decomposer_retty,
+    num_items,
+    begin_bit,
+    end_bit,
+    is_overwrite_okay,
+    &selector,
+    build_cache,
+    test_key);
+
+  assert(selector == 0 || selector == 1);
+
+  if (is_descending)
+  {
+    std::sort(expected_keys.begin(), expected_keys.end(), std::greater<KeyT>());
+  }
+  else
+  {
+    std::sort(expected_keys.begin(), expected_keys.end());
+  }
+
+  auto& output_keys = (is_overwrite_okay && selector == 0) ? input_keys_it : output_keys_it;
+  REQUIRE(expected_keys == std::vector<KeyT>(output_keys));
+}
+
+struct DeviceRadixSort_SortPairs_Fixture_Tag;
+C2H_TEST("DeviceRadixSort::SortPairs works", "[radix_sort]", test_params_tuple)
+{
+  using T     = c2h::get<0, TestType>;
+  using KeyT  = typename T::KeyT;
+  using ItemT = typename T::ItemT;
+
+  constexpr auto this_test_params = T();
+  const int num_items             = GENERATE_COPY(take(2, random(1, 1000000)), values({500, 1000000, 2000000}));
+  const bool is_descending        = this_test_params.is_descending();
+  const auto order                = is_descending ? CCCL_DESCENDING : CCCL_ASCENDING;
+
+  const int begin_bit          = 0;
+  const int end_bit            = sizeof(KeyT) * 8;
+  const bool is_overwrite_okay = this_test_params.is_overwrite_okay();
+  int selector                 = -1;
+
+  static constexpr cccl_op_t decomposer_no_op{};
+  static constexpr const char* unused_decomposer_retty = "";
+
+  // problem descriptor in this example: (order, TestType, item_t, is_overwrite_ok)
+
+  std::vector<KeyT> input_keys = make_shuffled_sequence<KeyT>(num_items);
+  std::vector<ItemT> input_items(num_items);
+  std::transform(input_keys.begin(), input_keys.end(), input_items.begin(), [](KeyT key) {
+    return static_cast<ItemT>(key);
+  });
+
+  std::vector<KeyT> expected_keys   = input_keys;
+  std::vector<ItemT> expected_items = input_items;
+
+  pointer_t<KeyT> input_keys_it(input_keys);
+  pointer_t<KeyT> output_keys_it(num_items);
+
+  pointer_t<ItemT> input_items_it(input_items);
+  pointer_t<ItemT> output_items_it(num_items);
+
+  auto& build_cache = get_cache<DeviceRadixSort_SortPairs_Fixture_Tag>();
+
+  const std::string& key_string = KeyBuilder::join(
+    {KeyBuilder::bool_as_key(is_descending),
+     KeyBuilder::type_as_key<KeyT>(),
+     KeyBuilder::type_as_key<ItemT>(),
+     KeyBuilder::bool_as_key(is_overwrite_okay)});
+  const auto& test_key = std::make_optional(key_string);
+
+  radix_sort<false>(
+    order,
+    input_keys_it,
+    output_keys_it,
+    input_items_it,
+    output_items_it,
+    decomposer_no_op,
+    unused_decomposer_retty,
+    num_items,
+    begin_bit,
+    end_bit,
+    is_overwrite_okay,
+    &selector,
+    build_cache,
+    test_key);
+
+  assert(selector == 0 || selector == 1);
+
+  if (is_descending)
+  {
+    std::sort(expected_keys.begin(), expected_keys.end(), std::greater<KeyT>());
+    std::sort(expected_items.begin(), expected_items.end(), std::greater<ItemT>());
+  }
+  else
+  {
+    std::sort(expected_keys.begin(), expected_keys.end());
+    std::sort(expected_items.begin(), expected_items.end());
+  }
+
+  auto& output_keys  = (is_overwrite_okay && selector == 0) ? input_keys_it : output_keys_it;
+  auto& output_items = (is_overwrite_okay && selector == 0) ? input_items_it : output_items_it;
+  REQUIRE(expected_keys == std::vector<KeyT>(output_keys));
+  REQUIRE(expected_items == std::vector<ItemT>(output_items));
+}
+
+#ifndef CCCL_C_PARALLEL_V2
+C2H_TEST("RadixSort build result has serialization metadata populated", "[radix_sort][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  pointer_t<T> keys_in(1);
+  pointer_t<T> values_in(1);
+  static constexpr cccl_op_t decomposer_no_op{};
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_radix_sort_build(
+      &build,
+      CCCL_ASCENDING,
+      keys_in,
+      values_in,
+      decomposer_no_op,
+      "",
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path()));
+
+  CHECK(build.cc == build_info.get_cc_major() * 10 + build_info.get_cc_minor());
+  CHECK((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  CHECK(build.payload_size > 0);
+  CHECK(build.runtime_policy != nullptr);
+  CHECK(build.runtime_policy_size > 0);
+  REQUIRE(build.single_tile_kernel_lowered_name != nullptr);
+  CHECK(build.single_tile_kernel_lowered_name[0] != '\0');
+  REQUIRE(build.upsweep_kernel_lowered_name != nullptr);
+  CHECK(build.upsweep_kernel_lowered_name[0] != '\0');
+  REQUIRE(build.downsweep_kernel_lowered_name != nullptr);
+  CHECK(build.downsweep_kernel_lowered_name[0] != '\0');
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_radix_sort_cleanup(&build));
+}
+
+C2H_TEST("RadixSort compile/load round-trip", "[radix_sort][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  pointer_t<T> dummy_keys_in(1);
+  pointer_t<T> dummy_values_in(1);
+  static constexpr cccl_op_t decomposer_no_op{};
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_radix_sort_compile(
+      &build,
+      CCCL_ASCENDING,
+      dummy_keys_in,
+      dummy_values_in,
+      decomposer_no_op,
+      "",
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+
+  REQUIRE((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  REQUIRE(build.payload_size > 0);
+  REQUIRE(build.single_tile_kernel_lowered_name != nullptr);
+  REQUIRE(build.upsweep_kernel_lowered_name != nullptr);
+  REQUIRE(build.downsweep_kernel_lowered_name != nullptr);
+  CHECK(build.library == nullptr);
+  CHECK(build.single_tile_kernel == nullptr);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_radix_sort_load(&build));
+  REQUIRE(build.library != nullptr);
+  CHECK(build.single_tile_kernel != nullptr);
+  CHECK(build.upsweep_kernel != nullptr);
+  CHECK(build.downsweep_kernel != nullptr);
+
+  constexpr std::size_t n    = 16;
+  const std::vector<T> input = generate<T>(n);
+  pointer_t<T> keys_in(input);
+  pointer_t<T> keys_out(n);
+  pointer_t<T> values_in(n);
+  pointer_t<T> values_out(n);
+  CUstream null_stream      = nullptr;
+  size_t temp_storage_bytes = 0;
+  int selector              = -1;
+
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_radix_sort(
+      build,
+      nullptr,
+      &temp_storage_bytes,
+      keys_in,
+      keys_out,
+      values_in,
+      values_out,
+      decomposer_no_op,
+      n,
+      /*begin_bit=*/0,
+      /*end_bit=*/sizeof(T) * 8,
+      /*is_overwrite_okay=*/false,
+      &selector,
+      null_stream));
+  pointer_t<uint8_t> temp_storage(temp_storage_bytes);
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_radix_sort(
+      build,
+      temp_storage.ptr,
+      &temp_storage_bytes,
+      keys_in,
+      keys_out,
+      values_in,
+      values_out,
+      decomposer_no_op,
+      n,
+      /*begin_bit=*/0,
+      /*end_bit=*/sizeof(T) * 8,
+      /*is_overwrite_okay=*/false,
+      &selector,
+      null_stream));
+
+  std::vector<T> expected(input);
+  std::sort(expected.begin(), expected.end());
+  // is_overwrite_okay=false → result is always in keys_out (selector not used for routing)
+  REQUIRE(expected == std::vector<T>(keys_out));
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_radix_sort_cleanup(&build));
+}
+#endif // CCCL_C_PARALLEL_V2

cccl_upstream/c/parallel/test/test_reduce.cpp

@@ -0,0 +1,976 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <cstdint>
+#include <iostream> // std::cerr
+#include <memory>
+#include <numeric>
+#include <optional> // std::optional
+#include <string>
+#include <vector>
+
+#include <cuda.h>
+#include <cuda_runtime.h>
+
+#include "algorithm_execution.h"
+#include "build_result_caching.h"
+#include "test_util.h"
+#include <cccl/c/reduce.h>
+
+// serialize/deserialize is a v1-only feature; the header does not exist in
+// the v2 (HostJIT) include tree, and the tests using it are guarded the same way.
+#ifndef CCCL_C_PARALLEL_V2
+#  include <cccl/c/serialization.h>
+#endif
+
+using BuildResultT = cccl_device_reduce_build_result_t;
+
+struct reduce_cleanup
+{
+  CUresult operator()(BuildResultT* build_data) const noexcept
+  {
+    return cccl_device_reduce_cleanup(build_data);
+  }
+};
+
+using reduce_deleter       = BuildResultDeleter<BuildResultT, reduce_cleanup>;
+using reduce_build_cache_t = build_cache_t<std::string, result_wrapper_t<BuildResultT, reduce_deleter>>;
+
+template <typename Tag>
+auto& get_cache()
+{
+  return fixture<reduce_build_cache_t, Tag>::get_or_create().get_value();
+}
+
+struct reduce_build
+{
+  static bool should_check_sass(int cc_major)
+  {
+    // TODO: add a check for NVRTC version; ref nvbug 5243118
+    return cc_major < 9;
+  }
+
+  CUresult operator()(
+    BuildResultT* build_ptr,
+    cccl_determinism_t determinism,
+    cccl_iterator_t input,
+    cccl_iterator_t output,
+    uint64_t,
+    cccl_op_t op,
+    cccl_value_t init,
+    int cc_major,
+    int cc_minor,
+    const char* cub_path,
+    const char* thrust_path,
+    const char* libcudacxx_path,
+    const char* ctk_path) const noexcept
+  {
+    return cccl_device_reduce_build(
+      build_ptr,
+      input,
+      output,
+      op,
+      init,
+      determinism,
+      cc_major,
+      cc_minor,
+      cub_path,
+      thrust_path,
+      libcudacxx_path,
+      ctk_path);
+  }
+};
+
+struct reduce_build_ex
+{
+  static bool should_check_sass(int cc_major)
+  {
+    // TODO: add a check for NVRTC version; ref nvbug 5243118
+    return cc_major < 9;
+  }
+
+  cccl_build_config config;
+
+  reduce_build_ex(const char** extra_compile_flags, size_t num_flags, const char** extra_include_dirs, size_t num_dirs)
+      : config(make_build_config(extra_compile_flags, num_flags, extra_include_dirs, num_dirs))
+  {}
+
+  CUresult operator()(
+    BuildResultT* build_ptr,
+    cccl_determinism_t determinism,
+    cccl_iterator_t input,
+    cccl_iterator_t output,
+    uint64_t,
+    cccl_op_t op,
+    cccl_value_t init,
+    int cc_major,
+    int cc_minor,
+    const char* cub_path,
+    const char* thrust_path,
+    const char* libcudacxx_path,
+    const char* ctk_path) const noexcept
+  {
+    return cccl_device_reduce_build_ex(
+      build_ptr,
+      input,
+      output,
+      op,
+      init,
+      determinism,
+      cc_major,
+      cc_minor,
+      cub_path,
+      thrust_path,
+      libcudacxx_path,
+      ctk_path,
+      const_cast<cccl_build_config*>(&config));
+  }
+};
+
+struct reduce_run
+{
+  template <typename... Ts>
+  CUresult operator()(cccl_device_reduce_build_result_t build,
+                      void* d_temp_storage,
+                      size_t* temp_storage_bytes,
+                      cccl_determinism_t determinism,
+                      Ts... args) const noexcept
+  {
+    if (determinism == CCCL_NOT_GUARANTEED)
+    {
+      return cccl_device_reduce_nondeterministic(build, d_temp_storage, temp_storage_bytes, args...);
+    }
+    else
+    {
+      return cccl_device_reduce(build, d_temp_storage, temp_storage_bytes, args...);
+    }
+  }
+};
+
+template <typename BuildCache = reduce_build_cache_t, typename KeyT = std::string>
+void reduce(cccl_iterator_t input,
+            cccl_iterator_t output,
+            uint64_t num_items,
+            cccl_op_t op,
+            cccl_value_t init,
+            cccl_determinism_t determinism,
+            std::optional<BuildCache>& cache,
+            const std::optional<KeyT>& lookup_key)
+{
+  AlgorithmExecute<BuildResultT, reduce_build, reduce_cleanup, reduce_run, BuildCache, KeyT>(
+    cache, lookup_key, determinism, input, output, num_items, op, init);
+}
+
+// ===============
+//   Tests section
+// ===============
+
+using integral_types = c2h::type_list<int32_t, uint32_t, int64_t, uint64_t>;
+struct Reduce_IntegralTypes_Fixture_Tag;
+C2H_TEST("Reduce works with integral types", "[reduce]", integral_types)
+{
+  using T = c2h::get<0, TestType>;
+
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 24)));
+  operation_t op              = make_operation("op", get_reduce_op(get_type_info<T>().type));
+  const std::vector<T> input  = generate<T>(num_items);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(1);
+  value_t<T> init{T{42}};
+
+  auto& build_cache    = get_cache<Reduce_IntegralTypes_Fixture_Tag>();
+  const auto& test_key = make_key<T>();
+
+  reduce(input_ptr, output_ptr, num_items, op, init, CCCL_RUN_TO_RUN, build_cache, test_key);
+
+  const T output   = output_ptr[0];
+  const T expected = std::accumulate(input.begin(), input.end(), init.value);
+  REQUIRE(output == expected);
+}
+
+struct Reduce_IntegralTypes_WellKnown_Fixture_Tag;
+C2H_TEST("Reduce works with integral types with well-known operations", "[reduce][well_known]", integral_types)
+{
+  using T = c2h::get<0, TestType>;
+
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 24)));
+  cccl_op_t op                = make_well_known_binary_operation();
+  const std::vector<T> input  = generate<T>(num_items);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(1);
+  value_t<T> init{T{42}};
+
+  auto& build_cache    = get_cache<Reduce_IntegralTypes_WellKnown_Fixture_Tag>();
+  const auto& test_key = make_key<T>();
+
+  reduce(input_ptr, output_ptr, num_items, op, init, CCCL_RUN_TO_RUN, build_cache, test_key);
+
+  const T output   = output_ptr[0];
+  const T expected = std::accumulate(input.begin(), input.end(), init.value);
+  REQUIRE(output == expected);
+}
+
+struct pair
+{
+  short a;
+  size_t b;
+};
+
+struct Reduce_CustomTypes_Fixture_Tag;
+C2H_TEST("Reduce works with custom types", "[reduce]")
+{
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 24)));
+
+  operation_t op              = make_operation("op",
+                                  R"(struct pair { short a; size_t b; };
+extern "C" __device__ void op(void* lhs_ptr, void* rhs_ptr, void* out_ptr) {
+  pair* lhs = static_cast<pair*>(lhs_ptr);
+  pair* rhs = static_cast<pair*>(rhs_ptr);
+  pair* out = static_cast<pair*>(out_ptr);
+  *out = pair{ lhs->a + rhs->a, lhs->b + rhs->b };
+})");
+  const std::vector<short> a  = generate<short>(num_items);
+  const std::vector<size_t> b = generate<size_t>(num_items);
+  std::vector<pair> input(num_items);
+  for (std::size_t i = 0; i < num_items; ++i)
+  {
+    input[i] = pair{a[i], b[i]};
+  }
+  pointer_t<pair> input_ptr(input);
+  pointer_t<pair> output_ptr(1);
+  value_t<pair> init{pair{4, 2}};
+
+  auto& build_cache    = get_cache<Reduce_CustomTypes_Fixture_Tag>();
+  const auto& test_key = make_key<pair>();
+
+  reduce(input_ptr, output_ptr, num_items, op, init, CCCL_RUN_TO_RUN, build_cache, test_key);
+
+  const pair output   = output_ptr[0];
+  const pair expected = std::accumulate(input.begin(), input.end(), init.value, [](const pair& lhs, const pair& rhs) {
+    return pair{short(lhs.a + rhs.a), lhs.b + rhs.b};
+  });
+  REQUIRE(output.a == expected.a);
+  REQUIRE(output.b == expected.b);
+}
+
+struct Reduce_CustomTypes_WellKnown_Fixture_Tag;
+C2H_TEST("Reduce works with custom types with well-known operations", "[reduce][well_known]")
+{
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 24)));
+
+  operation_t op_state        = make_operation("op",
+                                        R"(struct pair { short a; size_t b; };
+extern "C" __device__ void op(void* lhs_ptr, void* rhs_ptr, void* out_ptr) {
+  pair* lhs = static_cast<pair*>(lhs_ptr);
+  pair* rhs = static_cast<pair*>(rhs_ptr);
+  pair* out = static_cast<pair*>(out_ptr);
+  *out = pair{ lhs->a + rhs->a, lhs->b + rhs->b };
+})");
+  cccl_op_t op                = op_state;
+  op.type                     = cccl_op_kind_t::CCCL_PLUS;
+  const std::vector<short> a  = generate<short>(num_items);
+  const std::vector<size_t> b = generate<size_t>(num_items);
+  std::vector<pair> input(num_items);
+  for (std::size_t i = 0; i < num_items; ++i)
+  {
+    input[i] = pair{a[i], b[i]};
+  }
+  pointer_t<pair> input_ptr(input);
+  pointer_t<pair> output_ptr(1);
+  value_t<pair> init{pair{4, 2}};
+
+  auto& build_cache    = get_cache<Reduce_CustomTypes_WellKnown_Fixture_Tag>();
+  const auto& test_key = make_key<pair>();
+
+  reduce(input_ptr, output_ptr, num_items, op, init, CCCL_RUN_TO_RUN, build_cache, test_key);
+
+  const pair output   = output_ptr[0];
+  const pair expected = std::accumulate(input.begin(), input.end(), init.value, [](const pair& lhs, const pair& rhs) {
+    return pair{short(lhs.a + rhs.a), lhs.b + rhs.b};
+  });
+  REQUIRE(output.a == expected.a);
+  REQUIRE(output.b == expected.b);
+}
+
+struct Reduce_InputIterators_Fixture_Tag;
+C2H_TEST("Reduce works with input iterators", "[reduce]")
+{
+  const std::size_t num_items = GENERATE(1, 42, take(4, random(1 << 12, 1 << 16)));
+  operation_t op              = make_operation("op", get_reduce_op(get_type_info<int>().type));
+  iterator_t<int, counting_iterator_state_t<int>> input_it = make_counting_iterator<int>("int");
+  input_it.state.value                                     = 0;
+  pointer_t<int> output_it(1);
+  value_t<int> init{42};
+
+  auto& build_cache    = get_cache<Reduce_CustomTypes_Fixture_Tag>();
+  const auto& test_key = make_key<int>();
+
+  reduce(input_it, output_it, num_items, op, init, CCCL_RUN_TO_RUN, build_cache, test_key);
+
+  const int output   = output_it[0];
+  const int expected = init.value + static_cast<int>(num_items * (num_items - 1) / 2);
+  REQUIRE(output == expected);
+}
+
+struct Reduce_OutputIterators_Fixture_Tag;
+C2H_TEST("Reduce works with output iterators", "[reduce]")
+{
+  const int num_items = GENERATE(1, 42, take(4, random(1 << 12, 1 << 16)));
+  operation_t op      = make_operation("op", get_reduce_op(get_type_info<int>().type));
+  iterator_t<int, random_access_iterator_state_t<int>> output_it =
+    make_random_access_iterator<int>(iterator_kind::OUTPUT, "int", "out", " * 2");
+  const std::vector<int> input = generate<int>(num_items);
+  pointer_t<int> input_it(input);
+  pointer_t<int> inner_output_it(1);
+  output_it.state.data = inner_output_it.ptr;
+  value_t<int> init{42};
+
+  auto& build_cache    = get_cache<Reduce_OutputIterators_Fixture_Tag>();
+  const auto& test_key = make_key<int>();
+
+  reduce(input_it, output_it, num_items, op, init, CCCL_RUN_TO_RUN, build_cache, test_key);
+
+  const int output   = inner_output_it[0];
+  const int expected = std::accumulate(input.begin(), input.end(), init.value);
+  REQUIRE(output == expected * 2);
+}
+
+struct Reduce_InputOutputIterators_Fixture_Tag;
+C2H_TEST("Reduce works with input and output iterators", "[reduce]")
+{
+  const int num_items = GENERATE(1, 42, take(4, random(1 << 12, 1 << 16)));
+  operation_t op      = make_operation("op", get_reduce_op(get_type_info<int>().type));
+  iterator_t<int, constant_iterator_state_t<int>> input_it = make_constant_iterator<int>("int");
+  input_it.state.value                                     = 1;
+  iterator_t<int, random_access_iterator_state_t<int>> output_it =
+    make_random_access_iterator<int>(iterator_kind::OUTPUT, "int", "out", " * 2");
+  pointer_t<int> inner_output_it(1);
+  output_it.state.data = inner_output_it.ptr;
+  value_t<int> init{42};
+
+  auto& build_cache    = get_cache<Reduce_InputOutputIterators_Fixture_Tag>();
+  const auto& test_key = make_key<int>();
+
+  reduce(input_it, output_it, num_items, op, init, CCCL_RUN_TO_RUN, build_cache, test_key);
+
+  const int output   = inner_output_it[0];
+  const int expected = 2 * (init.value + num_items);
+  REQUIRE(output == expected);
+}
+
+struct Reduce_AccumulatorType_Fixture_Tag;
+C2H_TEST("Reduce accumulator type is influenced by initial value", "[reduce]")
+{
+  const std::size_t num_items = 1 << 14; // 16384 > 128
+
+  operation_t op = make_operation("op", get_reduce_op(get_type_info<size_t>().type));
+  iterator_t<char, constant_iterator_state_t<char>> input_it = make_constant_iterator<char>("char");
+  input_it.state.value                                       = 1;
+  pointer_t<size_t> output_it(1);
+  value_t<size_t> init{42};
+
+  auto& build_cache    = get_cache<Reduce_AccumulatorType_Fixture_Tag>();
+  const auto& test_key = make_key<char, size_t>();
+
+  reduce(input_it, output_it, num_items, op, init, CCCL_RUN_TO_RUN, build_cache, test_key);
+
+  const size_t output   = output_it[0];
+  const size_t expected = init.value + num_items;
+  REQUIRE(output == expected);
+}
+
+C2H_TEST("Reduce works with large inputs", "[reduce]")
+{
+  const size_t num_items = 1ull << 33;
+  operation_t op         = make_operation("op", get_reduce_op(get_type_info<size_t>().type));
+  iterator_t<char, constant_iterator_state_t<char>> input_it = make_constant_iterator<char>("char");
+  input_it.state.value                                       = 1;
+  pointer_t<size_t> output_it(1);
+  value_t<size_t> init{42};
+
+  // reuse fixture cache from previous example, as it runs identical example on larger input
+  auto& build_cache    = get_cache<Reduce_AccumulatorType_Fixture_Tag>();
+  const auto& test_key = make_key<char, size_t>();
+
+  reduce(input_it, output_it, num_items, op, init, CCCL_RUN_TO_RUN, build_cache, test_key);
+
+  const size_t output   = output_it[0];
+  const size_t expected = init.value + num_items;
+  REQUIRE(output == expected);
+}
+
+struct invocation_counter_state_t
+{
+  int* d_counter;
+};
+
+C2H_TEST("Reduce works with stateful operators", "[reduce]")
+{
+  const int num_items = 1 << 12;
+  pointer_t<int> counter(1);
+  stateful_operation_t<invocation_counter_state_t> op = make_operation(
+    "op",
+    R"(struct invocation_counter_state_t { int* d_counter; };
+extern "C" __device__ void op(void* state_ptr, void* a_ptr, void* b_ptr, void* out_ptr) {
+  invocation_counter_state_t* state = static_cast<invocation_counter_state_t*>(state_ptr);
+  atomicAdd(state->d_counter, 1);
+  int a = *static_cast<int*>(a_ptr);
+  int b = *static_cast<int*>(b_ptr);
+  *static_cast<int*>(out_ptr) = a + b;
+})",
+    invocation_counter_state_t{counter.ptr});
+
+  const std::vector<int> input = generate<int>(num_items);
+  pointer_t<int> input_ptr(input);
+  pointer_t<int> output_ptr(1);
+  value_t<int> init{42};
+
+  // turn off caching, since the example is only compiled once
+  std::optional<reduce_build_cache_t> build_cache = std::nullopt;
+  std::optional<std::string> test_key             = std::nullopt;
+
+  reduce(input_ptr, output_ptr, num_items, op, init, CCCL_RUN_TO_RUN, build_cache, test_key);
+
+  const int invocation_count          = counter[0];
+  const int expected_invocation_count = num_items - 1;
+  REQUIRE(invocation_count > expected_invocation_count);
+
+  const int output   = output_ptr[0];
+  const int expected = std::accumulate(input.begin(), input.end(), init.value);
+  REQUIRE(output == expected);
+}
+
+C2H_TEST("Reduce works with C++ source operations", "[reduce]")
+{
+  using T = int32_t;
+
+  const std::size_t num_items = GENERATE(42, 1337, 42000);
+
+  // Create operation from C++ source instead of LTO-IR
+  std::string cpp_source = R"(
+    extern "C" __device__ void op(void* a, void* b, void* out) {
+      int* ia = (int*)a;
+      int* ib = (int*)b;
+      int* iout = (int*)out;
+      *iout = *ia + *ib;
+    }
+  )";
+
+  operation_t op = make_cpp_operation("op", cpp_source);
+
+  const std::vector<T> input = generate<T>(num_items);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(1);
+  value_t<T> init{T{0}};
+
+  // Test key including flag that this uses C++ source
+  std::optional<std::string> test_key = std::format("cpp_source_test_{}_{}", num_items, typeid(T).name());
+
+  auto& cache                                   = get_cache<Reduce_IntegralTypes_Fixture_Tag>();
+  std::optional<reduce_build_cache_t> cache_opt = cache;
+  reduce(input_ptr, output_ptr, num_items, op, init, CCCL_RUN_TO_RUN, cache_opt, test_key);
+
+  const T output   = output_ptr[0];
+  const T expected = std::accumulate(input.begin(), input.end(), init.value);
+  REQUIRE(output == expected);
+}
+
+struct Reduce_FloatingPointTypes_Fixture_Tag;
+using floating_point_types = c2h::type_list<
+#if _CCCL_HAS_NVFP16()
+  __half,
+#endif
+  float,
+  double>;
+C2H_TEST("Reduce works with floating point types", "[reduce]", floating_point_types)
+{
+  using T = c2h::get<0, TestType>;
+
+  // Use small input sizes and values to avoid floating point precision issues.
+  const std::size_t num_items = GENERATE(10, 42, 1025);
+  operation_t op              = make_operation("op", get_reduce_op(get_type_info<T>().type));
+  const std::vector<T> input(num_items, T{1});
+
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(1);
+  value_t<T> init{T{42}};
+
+  auto& build_cache    = get_cache<Reduce_FloatingPointTypes_Fixture_Tag>();
+  const auto& test_key = make_key<T>();
+
+  reduce(input_ptr, output_ptr, num_items, op, init, CCCL_RUN_TO_RUN, build_cache, test_key);
+
+  const T output   = output_ptr[0];
+  const T expected = std::accumulate(input.begin(), input.end(), init.value);
+  REQUIRE_APPROX_EQ(std::vector<T>{output}, std::vector<T>{expected});
+}
+
+struct Reduce_CppSourceWithEx_Fixture_Tag;
+C2H_TEST("Reduce works with C++ source operations using _ex build", "[reduce]")
+{
+  using T = int32_t;
+
+  const std::size_t num_items = GENERATE(42, 1337, 42000);
+
+  // Create operation from C++ source that uses the identity function from header
+  std::string cpp_source = R"(
+    #include "test_identity.h"
+    extern "C" __device__ void op(void* a, void* b, void* out) {
+      int* ia = (int*)a;
+      int* ib = (int*)b;
+      int* iout = (int*)out;
+      int val_a = test_identity(*ia);
+      int val_b = test_identity(*ib);
+      *iout = val_a + val_b;
+    }
+  )";
+
+  operation_t op = make_cpp_operation("op", cpp_source);
+
+  const std::vector<T> input = generate<T>(num_items);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(1);
+  value_t<T> init{T{0}};
+
+  // Prepare extra compile flags and include paths
+  const char* extra_flags[]    = {"-DTEST_IDENTITY_ENABLED"};
+  const char* extra_includes[] = {TEST_INCLUDE_PATH};
+
+  // Use extended AlgorithmExecute with custom build configuration
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  BuildResultT build{};
+  reduce_build_ex builder(extra_flags, 1, extra_includes, 1);
+
+  REQUIRE(
+    CUDA_SUCCESS
+    == builder(
+      &build,
+      CCCL_RUN_TO_RUN,
+      input_ptr,
+      output_ptr,
+      num_items,
+      op,
+      init,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path()));
+
+  CUstream null_stream      = nullptr;
+  size_t temp_storage_bytes = 0;
+  REQUIRE(CUDA_SUCCESS
+          == cccl_device_reduce(
+            build, nullptr, &temp_storage_bytes, input_ptr, output_ptr, num_items, op, init, null_stream));
+
+  pointer_t<uint8_t> temp_storage(temp_storage_bytes);
+  REQUIRE(CUDA_SUCCESS
+          == cccl_device_reduce(
+            build, temp_storage.ptr, &temp_storage_bytes, input_ptr, output_ptr, num_items, op, init, null_stream));
+
+  const T output   = output_ptr[0];
+  const T expected = std::accumulate(input.begin(), input.end(), init.value);
+  REQUIRE(output == expected);
+
+  // Cleanup
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_cleanup(&build));
+}
+
+#ifndef CCCL_C_PARALLEL_V2
+C2H_TEST("Reduce build result has serialization metadata populated", "[reduce][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  operation_t op             = make_operation("op", get_reduce_op(get_type_info<T>().type));
+  const std::vector<T> input = generate<T>(16);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(1);
+  value_t<T> init{T{0}};
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_reduce_build(
+      &build,
+      input_ptr,
+      output_ptr,
+      op,
+      init,
+      CCCL_RUN_TO_RUN,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path()));
+
+  // cc field is packed as cc_major * 10 + cc_minor
+  CHECK(build.cc == build_info.get_cc_major() * 10 + build_info.get_cc_minor());
+
+  CHECK((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  CHECK(build.payload_size > 0);
+
+  CHECK(build.runtime_policy != nullptr);
+
+  REQUIRE(build.single_tile_kernel_lowered_name != nullptr);
+  CHECK(build.single_tile_kernel_lowered_name[0] != '\0');
+  REQUIRE(build.single_tile_second_kernel_lowered_name != nullptr);
+  CHECK(build.single_tile_second_kernel_lowered_name[0] != '\0');
+  REQUIRE(build.reduction_kernel_lowered_name != nullptr);
+  CHECK(build.reduction_kernel_lowered_name[0] != '\0');
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_cleanup(&build));
+}
+
+struct Reduce_Nondeterministic_Plus_Fixture_Tag;
+C2H_TEST("Reduce works with not_guaranteed determinism and plus", "[reduce][nondeterministic]")
+{
+  using T = float;
+
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 24)));
+  cccl_op_t op                = make_well_known_binary_operation(); // plus
+  const std::vector<T> input(num_items, T{1});
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(1);
+  value_t<T> init{T{0}};
+
+  auto& build_cache    = get_cache<Reduce_Nondeterministic_Plus_Fixture_Tag>();
+  const auto& test_key = make_key<T>();
+
+  reduce(input_ptr, output_ptr, num_items, op, init, CCCL_NOT_GUARANTEED, build_cache, test_key);
+
+  const T output   = output_ptr[0];
+  const T expected = std::accumulate(input.begin(), input.end(), init.value);
+  REQUIRE(output == expected);
+}
+
+C2H_TEST("Reduce compile/load round-trip", "[reduce][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  cccl_op_t op = make_well_known_binary_operation(); // plus
+  pointer_t<T> dummy_in(1);
+  pointer_t<T> dummy_out(1);
+  value_t<T> init{T{0}};
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_reduce_compile(
+      &build,
+      dummy_in,
+      dummy_out,
+      op,
+      init,
+      CCCL_RUN_TO_RUN,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+
+  REQUIRE((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  REQUIRE(build.payload_size > 0);
+  REQUIRE(build.single_tile_kernel_lowered_name != nullptr);
+  REQUIRE(build.single_tile_second_kernel_lowered_name != nullptr);
+  REQUIRE(build.reduction_kernel_lowered_name != nullptr);
+  CHECK(build.library == nullptr);
+  CHECK(build.single_tile_kernel == nullptr);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_load(&build));
+
+  REQUIRE(build.library != nullptr);
+  CHECK(build.single_tile_kernel != nullptr);
+  CHECK(build.single_tile_second_kernel != nullptr);
+  CHECK(build.reduction_kernel != nullptr);
+
+  constexpr std::size_t n    = 16;
+  const std::vector<T> input = generate<T>(n);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(1);
+  CUstream null_stream      = nullptr;
+  size_t temp_storage_bytes = 0;
+
+  REQUIRE(CUDA_SUCCESS
+          == cccl_device_reduce(build, nullptr, &temp_storage_bytes, input_ptr, output_ptr, n, op, init, null_stream));
+
+  pointer_t<uint8_t> temp_storage(temp_storage_bytes);
+  REQUIRE(CUDA_SUCCESS
+          == cccl_device_reduce(
+            build, temp_storage.ptr, &temp_storage_bytes, input_ptr, output_ptr, n, op, init, null_stream));
+
+  const T result   = output_ptr[0];
+  const T expected = std::accumulate(input.begin(), input.end(), T{0});
+  REQUIRE(result == expected);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_cleanup(&build));
+}
+
+C2H_TEST("Reduce link_ltoir round-trip", "[reduce][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  // Kernel-only compile: op has a name but no LTOIR (code_size == 0).
+  // compile() will produce kernel LTOIR with an unresolved external reference to "op".
+  cccl_op_t op_ko{};
+  op_ko.type      = CCCL_STATELESS;
+  op_ko.name      = "op";
+  op_ko.code      = nullptr;
+  op_ko.code_size = 0;
+  op_ko.code_type = CCCL_OP_LTOIR;
+
+  pointer_t<T> dummy_in(1);
+  pointer_t<T> dummy_out(1);
+  value_t<T> init{T{0}};
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_reduce_compile(
+      &build,
+      dummy_in,
+      dummy_out,
+      op_ko,
+      init,
+      CCCL_RUN_TO_RUN,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+
+  // After kernel-only compile: kernel_ltoir is populated, cubin is not.
+  REQUIRE((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_LTOIR));
+  REQUIRE(build.payload_size > 0);
+  CHECK((build.payload_kind != CCCL_PAYLOAD_CUBIN));
+  CHECK(build.library == nullptr);
+
+  // Compile the operator LTOIR separately (this is the "user-supplied" op blob).
+  operation_t op_full = make_operation("op", get_reduce_op(get_type_info<T>().type));
+  const void* op_blob = op_full.code.data();
+  size_t op_size      = op_full.code.size();
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_link_ltoir(&build, &op_blob, &op_size, 1));
+  REQUIRE((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  REQUIRE(build.library == nullptr);
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_load(&build));
+  REQUIRE(build.library != nullptr);
+  CHECK((build.payload_kind != CCCL_PAYLOAD_LTOIR));
+
+  constexpr std::size_t n    = 16;
+  const std::vector<T> input = generate<T>(n);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(1);
+  CUstream null_stream      = nullptr;
+  size_t temp_storage_bytes = 0;
+
+  cccl_op_t op_run    = op_full;
+  cccl_value_t init_v = init;
+
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_reduce(build, nullptr, &temp_storage_bytes, input_ptr, output_ptr, n, op_run, init_v, null_stream));
+
+  pointer_t<uint8_t> temp_storage(temp_storage_bytes);
+  REQUIRE(CUDA_SUCCESS
+          == cccl_device_reduce(
+            build, temp_storage.ptr, &temp_storage_bytes, input_ptr, output_ptr, n, op_run, init_v, null_stream));
+
+  const T result   = output_ptr[0];
+  const T expected = std::accumulate(input.begin(), input.end(), T{0});
+  REQUIRE(result == expected);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_cleanup(&build));
+}
+
+C2H_TEST("Reduce serialize/deserialize round-trip (cubin)", "[reduce][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  cccl_op_t op = make_well_known_binary_operation(); // plus
+  pointer_t<T> dummy_in(1);
+  pointer_t<T> dummy_out(1);
+  value_t<T> init{T{0}};
+
+  // Compile (full op → cubin payload)
+  BuildResultT build_a{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_reduce_compile(
+      &build_a,
+      dummy_in,
+      dummy_out,
+      op,
+      init,
+      CCCL_RUN_TO_RUN,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+  REQUIRE((build_a.payload != nullptr && build_a.payload_kind == CCCL_PAYLOAD_CUBIN));
+
+  // Serialize
+  void* blob       = nullptr;
+  size_t blob_size = 0;
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_serialize(&build_a, &blob, &blob_size));
+  REQUIRE(blob != nullptr);
+  REQUIRE(blob_size > 0);
+
+  // Cleanup the original — proves the serialized blob is self-contained.
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_cleanup(&build_a));
+
+  // Deserialize into a fresh build and load.
+  BuildResultT build_b{};
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_deserialize(&build_b, blob, blob_size));
+  REQUIRE(build_b.payload != nullptr);
+  REQUIRE(build_b.payload_kind == CCCL_PAYLOAD_CUBIN);
+  REQUIRE(build_b.runtime_policy != nullptr);
+  REQUIRE(build_b.single_tile_kernel_lowered_name != nullptr);
+  CHECK(build_b.library == nullptr);
+  CHECK(build_b.single_tile_kernel == nullptr);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_load(&build_b));
+  REQUIRE(build_b.library != nullptr);
+
+  constexpr std::size_t n    = 16;
+  const std::vector<T> input = generate<T>(n);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(1);
+  CUstream null_stream      = nullptr;
+  size_t temp_storage_bytes = 0;
+
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_reduce(build_b, nullptr, &temp_storage_bytes, input_ptr, output_ptr, n, op, init, null_stream));
+  pointer_t<uint8_t> temp_storage(temp_storage_bytes);
+  REQUIRE(CUDA_SUCCESS
+          == cccl_device_reduce(
+            build_b, temp_storage.ptr, &temp_storage_bytes, input_ptr, output_ptr, n, op, init, null_stream));
+
+  const T result   = output_ptr[0];
+  const T expected = std::accumulate(input.begin(), input.end(), T{0});
+  REQUIRE(result == expected);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_cleanup(&build_b));
+  cccl_serialization_buffer_free(blob);
+}
+
+C2H_TEST("Reduce serialize/deserialize round-trip (ltoir + link_ltoir)", "[reduce][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  // Kernel-only compile produces an LTOIR payload.
+  cccl_op_t op_ko{};
+  op_ko.type      = CCCL_STATELESS;
+  op_ko.name      = "op";
+  op_ko.code      = nullptr;
+  op_ko.code_size = 0;
+  op_ko.code_type = CCCL_OP_LTOIR;
+
+  pointer_t<T> dummy_in(1);
+  pointer_t<T> dummy_out(1);
+  value_t<T> init{T{0}};
+
+  BuildResultT build_a{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_reduce_compile(
+      &build_a,
+      dummy_in,
+      dummy_out,
+      op_ko,
+      init,
+      CCCL_RUN_TO_RUN,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+  REQUIRE((build_a.payload != nullptr && build_a.payload_kind == CCCL_PAYLOAD_LTOIR));
+
+  void* blob       = nullptr;
+  size_t blob_size = 0;
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_serialize(&build_a, &blob, &blob_size));
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_cleanup(&build_a));
+
+  BuildResultT build_b{};
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_deserialize(&build_b, blob, blob_size));
+  REQUIRE(build_b.payload_kind == CCCL_PAYLOAD_LTOIR);
+
+  // Link in the operator LTOIR (as user-supplied) and load.
+  operation_t op_full = make_operation("op", get_reduce_op(get_type_info<T>().type));
+  const void* op_blob = op_full.code.data();
+  size_t op_size      = op_full.code.size();
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_link_ltoir(&build_b, &op_blob, &op_size, 1));
+  REQUIRE(build_b.payload_kind == CCCL_PAYLOAD_CUBIN);
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_load(&build_b));
+
+  constexpr std::size_t n    = 16;
+  const std::vector<T> input = generate<T>(n);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(1);
+  CUstream null_stream      = nullptr;
+  size_t temp_storage_bytes = 0;
+
+  cccl_op_t op_run = op_full;
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_reduce(build_b, nullptr, &temp_storage_bytes, input_ptr, output_ptr, n, op_run, init, null_stream));
+  pointer_t<uint8_t> temp_storage(temp_storage_bytes);
+  REQUIRE(CUDA_SUCCESS
+          == cccl_device_reduce(
+            build_b, temp_storage.ptr, &temp_storage_bytes, input_ptr, output_ptr, n, op_run, init, null_stream));
+
+  REQUIRE(output_ptr[0] == std::accumulate(input.begin(), input.end(), T{0}));
+  REQUIRE(CUDA_SUCCESS == cccl_device_reduce_cleanup(&build_b));
+  cccl_serialization_buffer_free(blob);
+}
+
+C2H_TEST("Reduce deserialize rejects bad blobs", "[reduce][serialization]")
+{
+  BuildResultT build{};
+
+  // Empty input.
+  REQUIRE(CUDA_ERROR_INVALID_VALUE == cccl_device_reduce_deserialize(&build, nullptr, 0));
+
+  // Garbage bytes that are large enough to read a header from.
+  const char garbage[64] = {0};
+  REQUIRE(CUDA_SUCCESS != cccl_device_reduce_deserialize(&build, garbage, sizeof(garbage)));
+  // On failure build is zeroed.
+  CHECK(build.payload == nullptr);
+}
+#endif // CCCL_C_PARALLEL_V2

cccl_upstream/c/parallel/test/test_segmented_sort.cpp

@@ -0,0 +1,881 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdlib>
+#include <optional> // std::optional
+#include <string>
+#include <type_traits>
+#include <vector>
+
+#include <cuda_runtime.h>
+
+#include "algorithm_execution.h"
+#include "build_result_caching.h"
+#include "test_util.h"
+#include <cccl/c/segmented_sort.h>
+#include <cccl/c/types.h>
+
+using key_types = c2h::type_list<uint8_t, int16_t, uint32_t, double>;
+using item_t    = float;
+
+using BuildResultT = cccl_device_segmented_sort_build_result_t;
+
+using SizeT = ptrdiff_t;
+
+struct segmented_sort_cleanup
+{
+  CUresult operator()(BuildResultT* build_data) const noexcept
+  {
+    return cccl_device_segmented_sort_cleanup(build_data);
+  }
+};
+
+using segmented_sort_deleter       = BuildResultDeleter<BuildResultT, segmented_sort_cleanup>;
+using segmented_sort_build_cache_t = build_cache_t<std::string, result_wrapper_t<BuildResultT, segmented_sort_deleter>>;
+
+template <typename KeyTy, bool descending = false, bool overwrite_okay = false>
+struct TestParameters
+{
+  using KeyT                             = KeyTy;
+  static constexpr bool m_descending     = descending;
+  static constexpr bool m_overwrite_okay = overwrite_okay;
+
+  constexpr TestParameters() = default;
+
+  constexpr bool is_descending() const
+  {
+    return m_descending;
+  }
+  constexpr bool is_overwrite_okay() const
+  {
+    return m_overwrite_okay;
+  }
+};
+
+using test_params_tuple =
+  c2h::type_list<TestParameters<c2h::get<0, key_types>, false, false>,
+                 TestParameters<c2h::get<1, key_types>, true, false>,
+                 TestParameters<c2h::get<2, key_types>, false, true>,
+                 TestParameters<c2h::get<3, key_types>, true, true>>;
+
+template <typename Tag>
+auto& get_cache()
+{
+  return fixture<segmented_sort_build_cache_t, Tag>::get_or_create().get_value();
+}
+
+template <bool DisableSassCheckOnSm120 = false>
+struct segmented_sort_build
+{
+  static bool should_check_sass(int cc_major)
+  {
+    return !(DisableSassCheckOnSm120 && cc_major >= 12);
+  }
+
+  CUresult operator()(
+    BuildResultT* build_ptr,
+    cccl_sort_order_t sort_order,
+    cccl_iterator_t keys_in,
+    cccl_iterator_t /*keys_out*/,
+    cccl_iterator_t values_in,
+    cccl_iterator_t /*values_out*/,
+    int64_t /*num_items*/,
+    int64_t /*num_segments*/,
+    cccl_iterator_t start_offsets,
+    cccl_iterator_t end_offsets,
+    bool /*is_overwrite_okay*/,
+    int* /*selector*/,
+    int cc_major,
+    int cc_minor,
+    const char* cub_path,
+    const char* thrust_path,
+    const char* libcudacxx_path,
+    const char* ctk_path) const noexcept
+  {
+    return cccl_device_segmented_sort_build(
+      build_ptr,
+      sort_order,
+      keys_in,
+      values_in,
+      start_offsets,
+      end_offsets,
+      cc_major,
+      cc_minor,
+      cub_path,
+      thrust_path,
+      libcudacxx_path,
+      ctk_path);
+  }
+};
+
+struct segmented_sort_run
+{
+  template <typename... Rest>
+  CUresult operator()(
+    BuildResultT build,
+    void* temp_storage,
+    size_t* temp_storage_bytes,
+    cccl_sort_order_t,
+    cccl_iterator_t d_keys_in,
+    cccl_iterator_t d_keys_out,
+    cccl_iterator_t d_values_in,
+    cccl_iterator_t d_values_out,
+    int64_t num_items,
+    int64_t num_segments,
+    cccl_iterator_t start_offsets,
+    cccl_iterator_t end_offsets,
+    Rest... rest) const noexcept
+  {
+    return cccl_device_segmented_sort(
+      build,
+      temp_storage,
+      temp_storage_bytes,
+      d_keys_in,
+      d_keys_out,
+      d_values_in,
+      d_values_out,
+      num_items,
+      num_segments,
+      start_offsets,
+      end_offsets,
+      rest...);
+  }
+};
+
+template <bool DisableSassCheckOnSm120 = false,
+          typename BuildCache          = segmented_sort_build_cache_t,
+          typename KeyT                = std::string>
+void segmented_sort(
+  cccl_sort_order_t sort_order,
+  cccl_iterator_t keys_in,
+  cccl_iterator_t keys_out,
+  cccl_iterator_t values_in,
+  cccl_iterator_t values_out,
+  int64_t num_items,
+  int64_t num_segments,
+  cccl_iterator_t start_offsets,
+  cccl_iterator_t end_offsets,
+  bool is_overwrite_okay,
+  int* selector,
+  std::optional<BuildCache>& cache,
+  const std::optional<KeyT>& lookup_key)
+{
+  AlgorithmExecute<BuildResultT,
+                   segmented_sort_build<DisableSassCheckOnSm120>,
+                   segmented_sort_cleanup,
+                   segmented_sort_run,
+                   BuildCache,
+                   KeyT>(
+    cache,
+    lookup_key,
+    sort_order,
+    keys_in,
+    keys_out,
+    values_in,
+    values_out,
+    num_items,
+    num_segments,
+    start_offsets,
+    end_offsets,
+    is_overwrite_okay,
+    selector);
+}
+
+// ==============
+//   Test section
+// ==============
+
+struct SegmentedSort_KeysOnly_Fixture_Tag;
+C2H_TEST("segmented_sort can sort keys-only", "[segmented_sort][keys_only]", test_params_tuple)
+{
+  using T     = c2h::get<0, TestType>;
+  using key_t = typename T::KeyT;
+
+  constexpr auto this_test_params            = T();
+  constexpr bool is_descending               = this_test_params.is_descending();
+  constexpr auto order                       = is_descending ? CCCL_DESCENDING : CCCL_ASCENDING;
+  constexpr bool is_overwrite_okay           = this_test_params.is_overwrite_okay();
+  constexpr bool disable_sass_check_on_sm120 = std::is_same_v<key_t, c2h::get<3, key_types>>;
+
+  const std::size_t n_segments   = GENERATE(0, 13, take(2, random(1 << 10, 1 << 12)));
+  const std::size_t segment_size = GENERATE(1, 12, take(2, random(1 << 10, 1 << 12)));
+
+  const std::size_t n_elems = n_segments * segment_size;
+
+  std::vector<int> host_keys_int = generate<int>(n_elems);
+  std::vector<key_t> host_keys(n_elems);
+  std::transform(host_keys_int.begin(), host_keys_int.end(), host_keys.begin(), [](int val) {
+    return static_cast<key_t>(val);
+  });
+  std::vector<key_t> host_keys_out(n_elems);
+
+  REQUIRE(host_keys.size() == n_elems);
+  REQUIRE(host_keys_out.size() == n_elems);
+
+  pointer_t<key_t> keys_in_ptr(host_keys);
+  pointer_t<key_t> keys_out_ptr(host_keys_out);
+
+  pointer_t<item_t> values_in;
+  pointer_t<item_t> values_out;
+
+  // TODO: Using a step counting iterator does not work right now.
+  // static constexpr std::string_view index_ty_name = "signed long long";
+
+  // struct segment_offset_iterator_state_t
+  // {
+  //   SizeT linear_id;
+  //   SizeT segment_size;
+  // };
+
+  // static constexpr std::string_view offset_iterator_state_name = "segment_offset_iterator_state_t";
+  // static constexpr std::string_view advance_offset_method_name = "advance_offset_it";
+  // static constexpr std::string_view deref_offset_method_name   = "dereference_offset_it";
+
+  // const auto& [offset_iterator_state_src, offset_iterator_advance_src, offset_iterator_deref_src] =
+  //   make_step_counting_iterator_sources(
+  //     index_ty_name, offset_iterator_state_name, advance_offset_method_name, deref_offset_method_name);
+
+  // iterator_t<SizeT, segment_offset_iterator_state_t> start_offset_it =
+  //   make_iterator<SizeT, segment_offset_iterator_state_t>(
+  //     {offset_iterator_state_name, offset_iterator_state_src},
+  //     {advance_offset_method_name, offset_iterator_advance_src},
+  //     {deref_offset_method_name, offset_iterator_deref_src});
+
+  // start_offset_it.state.linear_id    = 0;
+  // start_offset_it.state.segment_size = segment_size;
+
+  // // Create end offset iterator (points to one past start)
+  // iterator_t<SizeT, segment_offset_iterator_state_t> end_offset_it =
+  //   make_iterator<SizeT, segment_offset_iterator_state_t>(
+  //     {offset_iterator_state_name, ""}, {advance_offset_method_name, ""}, {deref_offset_method_name, ""});
+
+  // end_offset_it.state.linear_id    = 1;
+  // end_offset_it.state.segment_size = segment_size;
+
+  // // Provide host-advance callbacks for offset iterators
+  // auto start_offsets_cccl         = static_cast<cccl_iterator_t>(start_offset_it);
+  // auto end_offsets_cccl           = static_cast<cccl_iterator_t>(end_offset_it);
+  // start_offsets_cccl.host_advance = &host_advance_linear_id<segment_offset_iterator_state_t>;
+  // end_offsets_cccl.host_advance   = &host_advance_linear_id<segment_offset_iterator_state_t>;
+
+  std::vector<SizeT> start_offsets(n_segments);
+  std::vector<SizeT> end_offsets(n_segments);
+  for (std::size_t i = 0; i < n_segments; ++i)
+  {
+    start_offsets[i] = static_cast<SizeT>(i * segment_size);
+    end_offsets[i]   = static_cast<SizeT>((i + 1) * segment_size);
+  }
+
+  pointer_t<SizeT> start_offsets_ptr(start_offsets);
+  pointer_t<SizeT> end_offsets_ptr(end_offsets);
+
+  auto& build_cache             = get_cache<SegmentedSort_KeysOnly_Fixture_Tag>();
+  const std::string& key_string = KeyBuilder::join(
+    {KeyBuilder::bool_as_key(is_descending),
+     KeyBuilder::type_as_key<key_t>(),
+     KeyBuilder::bool_as_key(is_overwrite_okay)});
+  const auto& test_key = std::make_optional(key_string);
+
+  int selector = -1;
+
+  segmented_sort<disable_sass_check_on_sm120>(
+    order,
+    keys_in_ptr,
+    keys_out_ptr,
+    values_in,
+    values_out,
+    n_elems,
+    n_segments,
+    // start_offsets_cccl,
+    // end_offsets_cccl,
+    start_offsets_ptr,
+    end_offsets_ptr,
+    is_overwrite_okay,
+    &selector,
+    build_cache,
+    test_key);
+
+  // Create expected result by sorting each segment
+  std::vector<key_t> expected_keys = host_keys;
+  for (std::size_t i = 0; i < n_segments; ++i)
+  {
+    std::size_t segment_start = i * segment_size;
+    std::size_t segment_end   = segment_start + segment_size;
+    if (is_descending)
+    {
+      std::sort(expected_keys.begin() + segment_start, expected_keys.begin() + segment_end, std::greater<key_t>());
+    }
+    else
+    {
+      std::sort(expected_keys.begin() + segment_start, expected_keys.begin() + segment_end);
+    }
+  }
+
+  auto& output_keys = (is_overwrite_okay && selector == 0) ? keys_in_ptr : keys_out_ptr;
+  REQUIRE(expected_keys == std::vector<key_t>(output_keys));
+}
+
+struct SegmentedSort_KeyValuePairs_Fixture_Tag;
+C2H_TEST("segmented_sort can sort key-value pairs", "[segmented_sort][key_value]", test_params_tuple)
+{
+  using T     = c2h::get<0, TestType>;
+  using key_t = typename T::KeyT;
+
+  constexpr auto this_test_params            = T();
+  constexpr bool is_descending               = this_test_params.is_descending();
+  constexpr auto order                       = is_descending ? CCCL_DESCENDING : CCCL_ASCENDING;
+  constexpr bool is_overwrite_okay           = this_test_params.is_overwrite_okay();
+  constexpr bool disable_sass_check_on_sm120 = !std::is_same_v<key_t, c2h::get<0, key_types>>;
+
+  const std::size_t n_segments   = GENERATE(0, 13, take(2, random(1 << 10, 1 << 12)));
+  const std::size_t segment_size = GENERATE(1, 12, take(2, random(1 << 10, 1 << 12)));
+
+  const std::size_t n_elems = n_segments * segment_size;
+
+  std::vector<int> host_keys_int = generate<int>(n_elems);
+  std::vector<key_t> host_keys(n_elems);
+  std::transform(host_keys_int.begin(), host_keys_int.end(), host_keys.begin(), [](int val) {
+    return static_cast<key_t>(val);
+  });
+  std::vector<int> host_values_int = generate<int>(n_elems);
+  std::vector<item_t> host_values(n_elems);
+  std::transform(host_values_int.begin(), host_values_int.end(), host_values.begin(), [](int val) {
+    return static_cast<item_t>(val);
+  });
+
+  std::vector<key_t> host_keys_out(n_elems);
+  std::vector<item_t> host_values_out(n_elems);
+
+  REQUIRE(host_keys.size() == n_elems);
+  REQUIRE(host_values.size() == n_elems);
+
+  pointer_t<key_t> keys_in_ptr(host_keys);
+  pointer_t<key_t> keys_out_ptr(host_keys_out);
+
+  pointer_t<item_t> values_in_ptr(host_values);
+  pointer_t<item_t> values_out_ptr(host_values_out);
+
+  std::vector<SizeT> start_offsets(n_segments);
+  std::vector<SizeT> end_offsets(n_segments);
+  for (std::size_t i = 0; i < n_segments; ++i)
+  {
+    start_offsets[i] = static_cast<SizeT>(i * segment_size);
+    end_offsets[i]   = static_cast<SizeT>((i + 1) * segment_size);
+  }
+
+  pointer_t<SizeT> start_offsets_ptr(start_offsets);
+  pointer_t<SizeT> end_offsets_ptr(end_offsets);
+
+  auto& build_cache             = get_cache<SegmentedSort_KeyValuePairs_Fixture_Tag>();
+  const std::string& key_string = KeyBuilder::join(
+    {KeyBuilder::bool_as_key(is_descending),
+     KeyBuilder::type_as_key<key_t>(),
+     KeyBuilder::type_as_key<item_t>(),
+     KeyBuilder::bool_as_key(is_overwrite_okay),
+     KeyBuilder::bool_as_key(n_elems == 0)}); // this results in the values pointer being null which results in a keys
+                                              // only build
+  const auto& test_key = std::make_optional(key_string);
+
+  int selector = -1;
+
+  segmented_sort<disable_sass_check_on_sm120>(
+    order,
+    keys_in_ptr,
+    keys_out_ptr,
+    values_in_ptr,
+    values_out_ptr,
+    n_elems,
+    n_segments,
+    // start_offsets_cccl,
+    // end_offsets_cccl,
+    start_offsets_ptr,
+    end_offsets_ptr,
+    is_overwrite_okay,
+    &selector,
+    build_cache,
+    test_key);
+
+  // Create expected result by sorting each segment with key-value pairs
+  std::vector<std::pair<key_t, item_t>> key_value_pairs;
+  key_value_pairs.reserve(n_elems);
+  for (std::size_t i = 0; i < n_elems; ++i)
+  {
+    key_value_pairs.emplace_back(host_keys[i], host_values[i]);
+  }
+
+  std::vector<key_t> expected_keys(n_elems);
+  std::vector<item_t> expected_values(n_elems);
+
+  for (std::size_t i = 0; i < n_segments; ++i)
+  {
+    std::size_t segment_start = i * segment_size;
+    std::size_t segment_end   = segment_start + segment_size;
+
+    if (is_descending)
+    {
+      std::stable_sort(key_value_pairs.begin() + segment_start,
+                       key_value_pairs.begin() + segment_end,
+                       [](const auto& a, const auto& b) {
+                         return b.first < a.first;
+                       });
+    }
+    else
+    {
+      std::stable_sort(key_value_pairs.begin() + segment_start,
+                       key_value_pairs.begin() + segment_end,
+                       [](const auto& a, const auto& b) {
+                         return a.first < b.first;
+                       });
+    }
+
+    // Extract sorted keys and values
+    for (std::size_t j = segment_start; j < segment_end; ++j)
+    {
+      expected_keys[j]   = key_value_pairs[j].first;
+      expected_values[j] = key_value_pairs[j].second;
+    }
+  }
+
+  auto& output_keys = (is_overwrite_okay && selector == 0) ? keys_in_ptr : keys_out_ptr;
+  auto& output_vals = (is_overwrite_okay && selector == 0) ? values_in_ptr : values_out_ptr;
+  REQUIRE(expected_keys == std::vector<key_t>(output_keys));
+  REQUIRE(expected_values == std::vector<item_t>(output_vals));
+}
+
+// These tests with custom types are currently failing TODO: add issue
+#ifdef NEVER_DEFINED
+struct custom_pair
+{
+  int key;
+  size_t value;
+
+  bool operator==(const custom_pair& other) const
+  {
+    return key == other.key && value == other.value;
+  }
+};
+
+struct SegmentedSort_CustomTypes_Fixture_Tag;
+C2H_TEST("SegmentedSort works with custom types as values", "[segmented_sort][custom_types]", test_params_tuple)
+{
+  using T       = c2h::get<0, TestType>;
+  using key_t   = typename T::KeyT;
+  using value_t = custom_pair;
+
+  constexpr auto this_test_params  = T();
+  constexpr bool is_descending     = this_test_params.is_descending();
+  constexpr auto order             = is_descending ? CCCL_DESCENDING : CCCL_ASCENDING;
+  constexpr bool is_overwrite_okay = this_test_params.is_overwrite_okay();
+
+  const std::size_t n_segments   = GENERATE(0, 13, take(2, random(1 << 10, 1 << 12)));
+  const std::size_t segment_size = GENERATE(1, 12, take(2, random(1 << 10, 1 << 12)));
+
+  const std::size_t n_elems = n_segments * segment_size;
+
+  // Generate primitive keys
+  std::vector<int> host_keys_int = generate<int>(n_elems);
+  std::vector<key_t> host_keys(n_elems);
+  std::transform(host_keys_int.begin(), host_keys_int.end(), host_keys.begin(), [](int x) {
+    return static_cast<key_t>(x);
+  });
+
+  // Generate custom values
+  std::vector<value_t> host_values(n_elems);
+  for (std::size_t i = 0; i < n_elems; ++i)
+  {
+    host_values[i] = value_t{static_cast<int>(i % 1000), static_cast<std::size_t>(i % 100)};
+  }
+  std::vector<key_t> host_keys_out(n_elems);
+  std::vector<value_t> host_values_out(n_elems);
+
+  pointer_t<key_t> keys_in_ptr(host_keys);
+  pointer_t<key_t> keys_out_ptr(host_keys_out);
+  pointer_t<value_t> values_in_ptr(host_values);
+  pointer_t<value_t> values_out_ptr(host_values_out);
+
+  using SizeT = long;
+  std::vector<SizeT> segments(n_segments + 1);
+  for (std::size_t i = 0; i <= n_segments; ++i)
+  {
+    segments[i] = i * segment_size;
+  }
+
+  pointer_t<SizeT> offset_ptr(segments);
+
+  auto start_offset_it = static_cast<cccl_iterator_t>(offset_ptr);
+  auto end_offset_it   = start_offset_it;
+  end_offset_it.state  = offset_ptr.ptr + 1;
+
+  auto& build_cache             = get_cache<SegmentedSort_CustomTypes_Fixture_Tag>();
+  const std::string& key_string = KeyBuilder::join(
+    {KeyBuilder::bool_as_key(is_descending),
+     KeyBuilder::type_as_key<key_t>(),
+     KeyBuilder::type_as_key<value_t>(),
+     KeyBuilder::bool_as_key(is_overwrite_okay),
+     KeyBuilder::bool_as_key(n_elems == 0)});
+  const auto& test_key = std::make_optional(key_string);
+
+  int selector = -1;
+
+  segmented_sort(
+    order,
+    keys_in_ptr,
+    keys_out_ptr,
+    values_in_ptr,
+    values_out_ptr,
+    n_elems,
+    n_segments,
+    start_offset_it,
+    end_offset_it,
+    is_overwrite_okay,
+    &selector,
+    build_cache,
+    test_key);
+
+  // Create expected result
+  std::vector<std::pair<key_t, value_t>> key_value_pairs;
+  for (std::size_t i = 0; i < n_elems; ++i)
+  {
+    key_value_pairs.emplace_back(host_keys[i], host_values[i]);
+  }
+
+  std::vector<key_t> expected_keys(n_elems);
+  std::vector<value_t> expected_values(n_elems);
+
+  for (std::size_t i = 0; i < n_segments; ++i)
+  {
+    std::size_t segment_start = segments[i];
+    std::size_t segment_end   = segments[i + 1];
+
+    if (is_descending)
+    {
+      std::stable_sort(key_value_pairs.begin() + segment_start,
+                       key_value_pairs.begin() + segment_end,
+                       [](const auto& a, const auto& b) {
+                         return b.first < a.first;
+                       });
+    }
+    else
+    {
+      std::stable_sort(key_value_pairs.begin() + segment_start,
+                       key_value_pairs.begin() + segment_end,
+                       [](const auto& a, const auto& b) {
+                         return a.first < b.first;
+                       });
+    }
+
+    // Extract sorted keys and values
+    for (std::size_t j = segment_start; j < segment_end; ++j)
+    {
+      expected_keys[j]   = key_value_pairs[j].first;
+      expected_values[j] = key_value_pairs[j].second;
+    }
+  }
+
+  auto& output_keys = (is_overwrite_okay && selector == 0) ? keys_in_ptr : keys_out_ptr;
+  auto& output_vals = (is_overwrite_okay && selector == 0) ? values_in_ptr : values_out_ptr;
+
+  REQUIRE(expected_keys == std::vector<key_t>(output_keys));
+  REQUIRE(expected_values == std::vector<value_t>(output_vals));
+}
+#endif
+
+struct SegmentedSort_VariableSegments_Fixture_Tag;
+C2H_TEST("SegmentedSort works with variable segment sizes", "[segmented_sort][variable_segments]", test_params_tuple)
+{
+  using T     = c2h::get<0, TestType>;
+  using key_t = typename T::KeyT;
+
+  constexpr auto this_test_params  = T();
+  constexpr bool is_descending     = this_test_params.is_descending();
+  constexpr auto order             = is_descending ? CCCL_DESCENDING : CCCL_ASCENDING;
+  constexpr bool is_overwrite_okay = this_test_params.is_overwrite_okay();
+  constexpr bool disable_sass_check_on_sm120 =
+    std::is_same_v<key_t, c2h::get<1, key_types>> || std::is_same_v<key_t, c2h::get<2, key_types>>;
+
+  const std::size_t n_segments = GENERATE(20, 600);
+
+  // Create variable segment sizes
+  const std::vector<std::size_t> base_pattern = {
+    1, 5, 10, 20, 30, 50, 100, 3, 25, 600, 7, 18, 300, 4, 35, 9, 14, 700, 28, 11};
+  std::vector<std::size_t> segment_sizes;
+  segment_sizes.reserve(n_segments);
+  while (segment_sizes.size() < n_segments)
+  {
+    const std::size_t remaining  = n_segments - segment_sizes.size();
+    const std::size_t copy_count = std::min(remaining, base_pattern.size());
+    segment_sizes.insert(segment_sizes.end(), base_pattern.begin(), base_pattern.begin() + copy_count);
+  }
+  REQUIRE(segment_sizes.size() == n_segments);
+
+  std::size_t n_elems = std::accumulate(segment_sizes.begin(), segment_sizes.end(), 0ULL);
+
+  std::vector<int> host_keys_int = generate<int>(n_elems);
+  std::vector<key_t> host_keys(n_elems);
+  std::transform(host_keys_int.begin(), host_keys_int.end(), host_keys.begin(), [](int val) {
+    return static_cast<key_t>(val);
+  });
+
+  // Generate float values by first generating ints and then transforming
+  std::vector<int> host_values_int = generate<int>(n_elems);
+  std::vector<item_t> host_values(n_elems);
+  std::transform(host_values_int.begin(), host_values_int.end(), host_values.begin(), [](int val) {
+    return static_cast<item_t>(val);
+  });
+  std::vector<key_t> host_keys_out(n_elems);
+  std::vector<item_t> host_values_out(n_elems);
+
+  pointer_t<key_t> keys_in_ptr(host_keys);
+  pointer_t<key_t> keys_out_ptr(host_keys_out);
+  pointer_t<item_t> values_in_ptr(host_values);
+  pointer_t<item_t> values_out_ptr(host_values_out);
+
+  std::vector<SizeT> start_offsets(n_segments);
+  std::vector<SizeT> end_offsets(n_segments);
+  SizeT current_offset = 0;
+  for (std::size_t i = 0; i < n_segments; ++i)
+  {
+    start_offsets[i] = current_offset;
+    current_offset += static_cast<SizeT>(segment_sizes[i]);
+    end_offsets[i] = current_offset;
+  }
+
+  pointer_t<SizeT> start_offsets_ptr(start_offsets);
+  pointer_t<SizeT> end_offsets_ptr(end_offsets);
+
+  auto& build_cache             = get_cache<SegmentedSort_VariableSegments_Fixture_Tag>();
+  const std::string& key_string = KeyBuilder::join(
+    {KeyBuilder::bool_as_key(is_descending),
+     KeyBuilder::type_as_key<key_t>(),
+     KeyBuilder::type_as_key<item_t>(),
+     KeyBuilder::bool_as_key(is_overwrite_okay)});
+  const auto& test_key = std::make_optional(key_string);
+
+  int selector = -1;
+
+  segmented_sort<disable_sass_check_on_sm120>(
+    order,
+    keys_in_ptr,
+    keys_out_ptr,
+    values_in_ptr,
+    values_out_ptr,
+    n_elems,
+    n_segments,
+    start_offsets_ptr,
+    end_offsets_ptr,
+    is_overwrite_okay,
+    &selector,
+    build_cache,
+    test_key);
+
+  // Create expected result
+  std::vector<std::pair<key_t, item_t>> key_value_pairs;
+  key_value_pairs.reserve(n_elems);
+  for (std::size_t i = 0; i < n_elems; ++i)
+  {
+    key_value_pairs.emplace_back(host_keys[i], host_values[i]);
+  }
+
+  std::vector<key_t> expected_keys(n_elems);
+  std::vector<item_t> expected_values(n_elems);
+
+  for (std::size_t i = 0; i < n_segments; ++i)
+  {
+    std::size_t segment_start = start_offsets[i];
+    std::size_t segment_end   = end_offsets[i];
+
+    if (is_descending)
+    {
+      std::stable_sort(key_value_pairs.begin() + segment_start,
+                       key_value_pairs.begin() + segment_end,
+                       [](const auto& a, const auto& b) {
+                         return b.first < a.first;
+                       });
+    }
+    else
+    {
+      std::stable_sort(key_value_pairs.begin() + segment_start,
+                       key_value_pairs.begin() + segment_end,
+                       [](const auto& a, const auto& b) {
+                         return a.first < b.first;
+                       });
+    }
+
+    // Extract sorted keys and values
+    for (std::size_t j = segment_start; j < segment_end; ++j)
+    {
+      expected_keys[j]   = key_value_pairs[j].first;
+      expected_values[j] = key_value_pairs[j].second;
+    }
+  }
+
+  auto& output_keys = (is_overwrite_okay && selector == 0) ? keys_in_ptr : keys_out_ptr;
+  auto& output_vals = (is_overwrite_okay && selector == 0) ? values_in_ptr : values_out_ptr;
+  REQUIRE(expected_keys == std::vector<key_t>(output_keys));
+  REQUIRE(expected_values == std::vector<item_t>(output_vals));
+}
+
+#ifndef CCCL_C_PARALLEL_V2
+C2H_TEST("SegmentedSort build result has serialization metadata populated", "[segmented_sort][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  pointer_t<T> keys_in(1);
+  pointer_t<T> values_in(1);
+  pointer_t<T> begin_offsets(1);
+  pointer_t<T> end_offsets(1);
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_segmented_sort_build(
+      &build,
+      CCCL_ASCENDING,
+      keys_in,
+      values_in,
+      begin_offsets,
+      end_offsets,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path()));
+
+  CHECK(build.cc == build_info.get_cc_major() * 10 + build_info.get_cc_minor());
+  CHECK((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  CHECK(build.payload_size > 0);
+  CHECK(build.runtime_policy != nullptr);
+  CHECK(build.runtime_policy_size > 0);
+  CHECK(build.partition_runtime_policy != nullptr);
+  CHECK(build.partition_runtime_policy_size > 0);
+  REQUIRE(build.segmented_sort_fallback_kernel_lowered_name != nullptr);
+  CHECK(build.segmented_sort_fallback_kernel_lowered_name[0] != '\0');
+  REQUIRE(build.segmented_sort_kernel_small_lowered_name != nullptr);
+  CHECK(build.segmented_sort_kernel_small_lowered_name[0] != '\0');
+  REQUIRE(build.three_way_partition_init_kernel_lowered_name != nullptr);
+  CHECK(build.three_way_partition_init_kernel_lowered_name[0] != '\0');
+  REQUIRE(build.three_way_partition_kernel_lowered_name != nullptr);
+  CHECK(build.three_way_partition_kernel_lowered_name[0] != '\0');
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_segmented_sort_cleanup(&build));
+}
+
+C2H_TEST("SegmentedSort compile/load round-trip", "[segmented_sort][serialization]")
+{
+  using T   = int32_t;
+  using Off = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  pointer_t<T> dummy_keys_in(1);
+  pointer_t<T> dummy_values_in(1);
+  pointer_t<Off> dummy_begin_offsets(1);
+  pointer_t<Off> dummy_end_offsets(1);
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_segmented_sort_compile(
+      &build,
+      CCCL_ASCENDING,
+      dummy_keys_in,
+      dummy_values_in,
+      dummy_begin_offsets,
+      dummy_end_offsets,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+
+  REQUIRE((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  REQUIRE(build.payload_size > 0);
+  REQUIRE(build.segmented_sort_fallback_kernel_lowered_name != nullptr);
+  REQUIRE(build.segmented_sort_kernel_small_lowered_name != nullptr);
+  REQUIRE(build.three_way_partition_init_kernel_lowered_name != nullptr);
+  CHECK(build.library == nullptr);
+  CHECK(build.segmented_sort_fallback_kernel == nullptr);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_segmented_sort_load(&build));
+  REQUIRE(build.library != nullptr);
+  CHECK(build.segmented_sort_fallback_kernel != nullptr);
+  CHECK(build.segmented_sort_kernel_small != nullptr);
+  CHECK(build.three_way_partition_init_kernel != nullptr);
+  CHECK(build.three_way_partition_kernel != nullptr);
+
+  constexpr std::size_t n          = 8;
+  constexpr std::size_t n_segments = 2;
+  const std::vector<T> input       = generate<T>(n);
+  pointer_t<T> keys_in(input);
+  pointer_t<T> keys_out(n);
+  pointer_t<T> values_in(n); // items not checked — just keys
+  pointer_t<T> values_out(n);
+  const std::vector<Off> begin_offsets_host = {0, static_cast<Off>(n / 2)};
+  const std::vector<Off> end_offsets_host   = {static_cast<Off>(n / 2), static_cast<Off>(n)};
+  pointer_t<Off> begin_offsets_ptr(begin_offsets_host);
+  pointer_t<Off> end_offsets_ptr(end_offsets_host);
+  CUstream null_stream      = nullptr;
+  size_t temp_storage_bytes = 0;
+  int selector              = -1;
+
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_segmented_sort(
+      build,
+      nullptr,
+      &temp_storage_bytes,
+      keys_in,
+      keys_out,
+      values_in,
+      values_out,
+      n,
+      n_segments,
+      begin_offsets_ptr,
+      end_offsets_ptr,
+      /*is_overwrite_okay=*/false,
+      &selector,
+      null_stream));
+  pointer_t<uint8_t> temp_storage(temp_storage_bytes);
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_segmented_sort(
+      build,
+      temp_storage.ptr,
+      &temp_storage_bytes,
+      keys_in,
+      keys_out,
+      values_in,
+      values_out,
+      n,
+      n_segments,
+      begin_offsets_ptr,
+      end_offsets_ptr,
+      /*is_overwrite_okay=*/false,
+      &selector,
+      null_stream));
+
+  auto& output_keys = (selector == 0) ? keys_in : keys_out;
+  std::vector<T> result(output_keys);
+  // Each segment should be sorted ascending
+  CHECK(std::is_sorted(result.begin(), result.begin() + n / 2));
+  CHECK(std::is_sorted(result.begin() + n / 2, result.end()));
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_segmented_sort_cleanup(&build));
+}
+#endif // CCCL_C_PARALLEL_V2

cccl_upstream/c/parallel/test/test_three_way_partition.cpp

@@ -0,0 +1,771 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <algorithm>
+#include <cstdint>
+#include <iterator>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include <cuda_runtime.h>
+
+#include "algorithm_execution.h"
+#include "build_result_caching.h"
+#include "test_util.h"
+#include <cccl/c/three_way_partition.h>
+
+using BuildResultT = cccl_device_three_way_partition_build_result_t;
+
+struct three_way_partition_cleanup
+{
+  CUresult operator()(BuildResultT* build_data) const noexcept
+  {
+    return cccl_device_three_way_partition_cleanup(build_data);
+  }
+};
+
+using three_way_partition_deleter = BuildResultDeleter<BuildResultT, three_way_partition_cleanup>;
+using three_way_partition_build_cache_t =
+  build_cache_t<std::string, result_wrapper_t<BuildResultT, three_way_partition_deleter>>;
+
+template <typename KeyType, typename NumSelectedType>
+struct TestParameters
+{
+  using KeyT         = KeyType;
+  using NumSelectedT = NumSelectedType;
+};
+
+template <typename Tag>
+auto& get_cache()
+{
+  return fixture<three_way_partition_build_cache_t, Tag>::get_or_create().get_value();
+}
+
+template <bool DisableSassCheck = false, bool DisableSassCheckOnSm120 = false>
+struct three_way_partition_build
+{
+  template <typename... Rest>
+  CUresult operator()(
+    BuildResultT* build_ptr,
+    cccl_iterator_t d_in,
+    cccl_iterator_t d_first_part_out,
+    cccl_iterator_t d_second_part_out,
+    cccl_iterator_t d_unselected_out,
+    cccl_iterator_t d_num_selected_out,
+    cccl_op_t select_first_part_op,
+    cccl_op_t select_second_part_op,
+    int64_t /*num_items*/,
+    Rest... rest) const noexcept
+  {
+    return cccl_device_three_way_partition_build(
+      build_ptr,
+      d_in,
+      d_first_part_out,
+      d_second_part_out,
+      d_unselected_out,
+      d_num_selected_out,
+      select_first_part_op,
+      select_second_part_op,
+      rest...);
+  }
+
+  static constexpr bool should_check_sass(int cc_major)
+  {
+    return !DisableSassCheck && !(DisableSassCheckOnSm120 && cc_major >= 12);
+  }
+};
+
+struct three_way_partition_run
+{
+  template <typename... Args>
+  CUresult operator()(Args... args) const noexcept
+  {
+    return cccl_device_three_way_partition(args...);
+  }
+};
+
+// Host-side reference implementation using the C++ standard library
+template <typename T>
+struct three_way_partition_result_t
+{
+  three_way_partition_result_t() = delete;
+  explicit three_way_partition_result_t(std::size_t num_items)
+      : first_part(num_items)
+      , second_part(num_items)
+      , unselected(num_items)
+  {}
+  explicit three_way_partition_result_t(
+    std::vector<T> first,
+    std::vector<T> second,
+    std::vector<T> unselected,
+    std::size_t n_first,
+    std::size_t n_second,
+    std::size_t n_unselected)
+      : first_part(std::move(first))
+      , second_part(std::move(second))
+      , unselected(std::move(unselected))
+      , num_items_in_first_part(n_first)
+      , num_items_in_second_part(n_second)
+      , num_unselected_items(n_unselected)
+  {}
+
+  std::vector<T> first_part;
+  std::vector<T> second_part;
+  std::vector<T> unselected;
+
+  std::size_t num_items_in_first_part{};
+  std::size_t num_items_in_second_part{};
+  std::size_t num_unselected_items{};
+
+  bool operator==(const three_way_partition_result_t<T>& other) const
+  {
+    if (num_items_in_first_part != other.num_items_in_first_part
+        || num_items_in_second_part != other.num_items_in_second_part
+        || num_unselected_items != other.num_unselected_items)
+    {
+      return false;
+    }
+    return std::equal(first_part.begin(), first_part.begin() + num_items_in_first_part, other.first_part.begin())
+        && std::equal(second_part.begin(), second_part.begin() + num_items_in_second_part, other.second_part.begin())
+        && std::equal(unselected.begin(), unselected.begin() + num_unselected_items, other.unselected.begin());
+  }
+};
+
+template <typename T>
+struct greater_or_equal_t
+{
+  T compare;
+
+  explicit __host__ greater_or_equal_t(T compare)
+      : compare(compare)
+  {}
+
+  __device__ bool operator()(const T& a) const
+  {
+    return a >= compare;
+  }
+};
+
+template <typename T>
+struct less_than_t
+{
+  T compare;
+
+  explicit __host__ less_than_t(T compare)
+      : compare(compare)
+  {}
+
+  __device__ bool operator()(const T& a) const
+  {
+    return a < compare;
+  }
+};
+
+template <typename FirstPartSelectionOp, typename SecondPartSelectionOp, typename T>
+three_way_partition_result_t<T>
+std_partition(FirstPartSelectionOp first_selector, SecondPartSelectionOp second_selector, const std::vector<T>& in)
+{
+  const int num_items = static_cast<int>(in.size());
+  three_way_partition_result_t<T> result(num_items);
+
+  std::vector<T> intermediate_result(num_items);
+
+  auto intermediate_iterators =
+    std::partition_copy(in.begin(), in.end(), result.first_part.begin(), intermediate_result.begin(), first_selector);
+
+  result.num_items_in_first_part =
+    static_cast<int>(std::distance(result.first_part.begin(), intermediate_iterators.first));
+
+  auto final_iterators = std::partition_copy(
+    intermediate_result.begin(),
+    intermediate_result.begin() + (num_items - result.num_items_in_first_part),
+    result.second_part.begin(),
+    result.unselected.begin(),
+    second_selector);
+
+  result.num_items_in_second_part = static_cast<int>(std::distance(result.second_part.begin(), final_iterators.first));
+  result.num_unselected_items     = static_cast<int>(std::distance(result.unselected.begin(), final_iterators.second));
+
+  return result;
+}
+
+template <typename OperationT,
+          typename KeyT,
+          typename NumSelectedT,
+          typename TagT,
+          bool DisableSassCheck        = false,
+          bool DisableSassCheckOnSm120 = false>
+three_way_partition_result_t<KeyT>
+c_parallel_partition(OperationT first_selector, OperationT second_selector, const std::vector<KeyT>& input)
+{
+  std::size_t num_items = input.size();
+
+  pointer_t<KeyT> input_ptr(input);
+  pointer_t<KeyT> first_part_output_ptr(num_items);
+  pointer_t<KeyT> second_part_output_ptr(num_items);
+  pointer_t<KeyT> unselected_output_ptr(num_items);
+  pointer_t<NumSelectedT> num_selected_ptr(2);
+
+  auto& build_cache    = get_cache<TagT>();
+  const auto& test_key = make_key<KeyT, NumSelectedT>();
+
+  three_way_partition<DisableSassCheck, DisableSassCheckOnSm120>(
+    input_ptr,
+    first_part_output_ptr,
+    second_part_output_ptr,
+    unselected_output_ptr,
+    num_selected_ptr,
+    first_selector,
+    second_selector,
+    num_items,
+    build_cache,
+    test_key);
+
+  std::vector<KeyT> first_part_output(first_part_output_ptr);
+  std::vector<KeyT> second_part_output(second_part_output_ptr);
+  std::vector<KeyT> unselected_output(unselected_output_ptr);
+  std::vector<NumSelectedT> num_selected(num_selected_ptr);
+
+  return three_way_partition_result_t<KeyT>(
+    std::move(first_part_output),
+    std::move(second_part_output),
+    std::move(unselected_output),
+    num_selected[0],
+    num_selected[1],
+    num_items - num_selected[0] - num_selected[1]);
+}
+
+template <bool DisableSassCheck        = false,
+          bool DisableSassCheckOnSm120 = false,
+          typename BuildCache          = three_way_partition_build_cache_t,
+          typename KeyT                = std::string>
+void three_way_partition(
+  cccl_iterator_t d_in,
+  cccl_iterator_t d_first_part_out,
+  cccl_iterator_t d_second_part_out,
+  cccl_iterator_t d_unselected_out,
+  cccl_iterator_t d_num_selected_out,
+  cccl_op_t select_first_part_op,
+  cccl_op_t select_second_part_op,
+  int64_t num_items,
+  std::optional<BuildCache>& cache,
+  const std::optional<KeyT>& lookup_key)
+{
+  AlgorithmExecute<BuildResultT,
+                   three_way_partition_build<DisableSassCheck, DisableSassCheckOnSm120>,
+                   three_way_partition_cleanup,
+                   three_way_partition_run,
+                   BuildCache,
+                   KeyT>(
+    cache,
+    lookup_key,
+    d_in,
+    d_first_part_out,
+    d_second_part_out,
+    d_unselected_out,
+    d_num_selected_out,
+    select_first_part_op,
+    select_second_part_op,
+    num_items);
+}
+
+// ==============
+//   Test section
+// ==============
+
+using key_types =
+  c2h::type_list<uint8_t,
+                 int16_t,
+                 uint32_t,
+                 int64_t,
+                 uint64_t,
+#if _CCCL_HAS_NVFP16()
+                 __half,
+#endif
+                 float,
+                 double>;
+
+using num_selected_types = c2h::type_list<uint32_t, int64_t>;
+
+using test_params_tuple =
+  c2h::type_list<TestParameters<c2h::get<0, key_types>, c2h::get<0, num_selected_types>>,
+                 TestParameters<c2h::get<1, key_types>, c2h::get<1, num_selected_types>>,
+                 TestParameters<c2h::get<2, key_types>, c2h::get<0, num_selected_types>>,
+                 TestParameters<c2h::get<3, key_types>, c2h::get<1, num_selected_types>>,
+                 TestParameters<c2h::get<4, key_types>, c2h::get<0, num_selected_types>>,
+                 TestParameters<c2h::get<5, key_types>, c2h::get<1, num_selected_types>>>;
+
+struct ThreeWayPartition_PrimitiveTypes_Fixture_Tag;
+C2H_TEST("ThreeWayPartition works with primitive types", "[three_way_partition]", test_params_tuple)
+{
+  using T              = c2h::get<0, TestType>;
+  using key_t          = T::KeyT;
+  using num_selected_t = T::NumSelectedT;
+
+  auto [less_op_src, greater_or_equal_op_src] = get_three_way_partition_ops(get_type_info<key_t>().type, 21);
+  operation_t less_op                         = make_operation("less_op", less_op_src);
+  operation_t greater_or_equal_op             = make_operation("greater_op", greater_or_equal_op_src);
+
+  const std::size_t num_items      = GENERATE(0, 42, take(4, random(1 << 12, 1 << 20)));
+  const std::vector<int> input_int = generate<int>(num_items);
+  const std::vector<key_t> input(input_int.begin(), input_int.end());
+
+  auto c_parallel_result =
+    c_parallel_partition<operation_t, key_t, num_selected_t, ThreeWayPartition_PrimitiveTypes_Fixture_Tag, true>(
+      less_op, greater_or_equal_op, input);
+  auto std_result = std_partition(less_than_t<key_t>{key_t{21}}, greater_or_equal_t<key_t>{key_t{21}}, input);
+
+  REQUIRE(c_parallel_result == std_result);
+}
+
+C2H_TEST("ThreeWayPartition works with Boolean well-known operations", "[three_way_partition][well_known]")
+{
+  const std::vector<uint8_t> input{1, 0, 0, 1, 1, 0};
+  const std::size_t num_items = input.size();
+
+  pointer_t<uint8_t> input_ptr(input);
+  pointer_t<uint8_t> first_part_output_ptr(num_items);
+  pointer_t<uint8_t> second_part_output_ptr(num_items);
+  pointer_t<uint8_t> unselected_output_ptr(num_items);
+  pointer_t<int64_t> num_selected_ptr(2);
+
+  cccl_op_t identity_op    = make_well_known_unary_operation();
+  identity_op.type         = cccl_op_kind_t::CCCL_IDENTITY;
+  cccl_op_t logical_not_op = make_well_known_unary_operation();
+  logical_not_op.type      = cccl_op_kind_t::CCCL_LOGICAL_NOT;
+
+  std::optional<three_way_partition_build_cache_t> no_cache = std::nullopt;
+  const std::optional<std::string> no_key                   = std::nullopt;
+  three_way_partition(
+    make_boolean_iterator(input_ptr),
+    make_boolean_iterator(first_part_output_ptr),
+    make_boolean_iterator(second_part_output_ptr),
+    make_boolean_iterator(unselected_output_ptr),
+    num_selected_ptr,
+    identity_op,
+    logical_not_op,
+    static_cast<int64_t>(num_items),
+    no_cache,
+    no_key);
+
+  const std::vector<int64_t> num_selected(num_selected_ptr);
+  REQUIRE(num_selected == std::vector<int64_t>{3, 3});
+
+  const std::vector<uint8_t> first_part_output(first_part_output_ptr);
+  const std::vector<uint8_t> second_part_output(second_part_output_ptr);
+  const std::vector<uint8_t> unselected_output(unselected_output_ptr);
+  const std::size_t num_unselected = num_items - static_cast<std::size_t>(num_selected[0] + num_selected[1]);
+  CHECK(std::vector<uint8_t>(first_part_output.begin(), first_part_output.begin() + 3)
+        == std::vector<uint8_t>{1, 1, 1});
+  CHECK(std::vector<uint8_t>(second_part_output.begin(), second_part_output.begin() + 3)
+        == std::vector<uint8_t>{0, 0, 0});
+  CHECK(std::vector<uint8_t>(unselected_output.begin(), unselected_output.begin() + num_unselected).empty());
+  CHECK(num_unselected == 0);
+}
+
+struct selector_state_t
+{
+  int comparison_value;
+};
+
+struct ThreeWayPartition_StatefulOperations_Fixture_Tag;
+C2H_TEST("ThreeWayPartition works with stateful operations", "[three_way_partition]")
+{
+  using key_t          = int;
+  using num_selected_t = int;
+
+  selector_state_t op_state                      = {21};
+  stateful_operation_t<selector_state_t> less_op = make_operation(
+    "less_op",
+    R"(struct selector_state_t { int comparison_value; };
+extern "C" __device__ void less_op(void* state_ptr, void* x_ptr, void* out_ptr) {
+  selector_state_t* state = static_cast<selector_state_t*>(state_ptr);
+  *static_cast<int*>(x_ptr) < state->comparison_value;
+  *static_cast<bool*>(out_ptr) = *static_cast<int*>(x_ptr) < state->comparison_value;
+})",
+    op_state);
+  stateful_operation_t<selector_state_t> greater_or_equal_op = make_operation(
+    "greater_or_equal_op",
+    R"(struct selector_state_t { int comparison_value; };
+extern "C" __device__ void greater_or_equal_op(void* state_ptr, void* x_ptr, void* out_ptr) {
+  selector_state_t* state = static_cast<selector_state_t*>(state_ptr);
+  *static_cast<int*>(x_ptr) >= state->comparison_value;
+  *static_cast<bool*>(out_ptr) = *static_cast<int*>(x_ptr) >= state->comparison_value;
+})",
+    op_state);
+
+  const std::size_t num_items      = GENERATE(0, 42, take(4, random(1 << 12, 1 << 20)));
+  const std::vector<int> input_int = generate<int>(num_items);
+  const std::vector<key_t> input(input_int.begin(), input_int.end());
+
+  auto c_parallel_result =
+    c_parallel_partition<stateful_operation_t<selector_state_t>,
+                         key_t,
+                         num_selected_t,
+                         ThreeWayPartition_StatefulOperations_Fixture_Tag,
+                         false,
+                         true>(less_op, greater_or_equal_op, input);
+  auto std_result = std_partition(less_than_t<key_t>{key_t{21}}, greater_or_equal_t<key_t>{key_t{21}}, input);
+
+  REQUIRE(c_parallel_result == std_result);
+}
+
+struct ThreeWayPartition_CustomTypes_Fixture_Tag;
+C2H_TEST("ThreeWayPartition works with custom types", "[three_way_partition]")
+{
+  struct pair_type
+  {
+    int a;
+    size_t b;
+
+    bool operator==(const pair_type& other) const
+    {
+      return a == other.a && b == other.b;
+    }
+  };
+
+  struct custom_greater_or_equal_t
+  {
+    int compare;
+
+    explicit __host__ custom_greater_or_equal_t(int compare)
+        : compare(compare)
+    {}
+
+    __device__ bool operator()(const pair_type& a) const
+    {
+      return a.a >= compare;
+    }
+  };
+
+  struct custom_less_than_t
+  {
+    int compare;
+
+    explicit __host__ custom_less_than_t(int compare)
+        : compare(compare)
+    {}
+
+    __device__ bool operator()(const pair_type& a) const
+    {
+      return a.a < compare;
+    }
+  };
+
+  using key_t          = pair_type;
+  using num_selected_t = int;
+
+  const int comparison_value = 21;
+
+  operation_t less_op = make_operation(
+    "less_op",
+    std::format(R"(struct pair_type {{ int a; size_t b; }};
+extern "C" __device__ void less_op(void* x_ptr, void* out_ptr) {{
+  pair_type* x = static_cast<pair_type*>(x_ptr);
+  bool* out = static_cast<bool*>(out_ptr);
+  *out = x->a < {0};
+}})",
+                comparison_value));
+  operation_t greater_or_equal_op = make_operation(
+    "greater_or_equal_op",
+    std::format(R"(struct pair_type {{ int a; size_t b; }};
+extern "C" __device__ void greater_or_equal_op(void* x_ptr, void* out_ptr) {{
+  pair_type* x = static_cast<pair_type*>(x_ptr);
+  bool* out = static_cast<bool*>(out_ptr);
+  *out = x->a >= {0};
+}})",
+                comparison_value));
+
+  const std::size_t num_items      = GENERATE(0, 42, take(4, random(1 << 12, 1 << 20)));
+  const std::vector<int> input_int = generate<int>(num_items);
+  std::vector<key_t> input(num_items);
+  std::transform(input_int.begin(), input_int.end(), input.begin(), [](const int& x) {
+    return key_t{static_cast<int>(x), static_cast<size_t>(x)};
+  });
+
+  auto c_parallel_result =
+    c_parallel_partition<operation_t, key_t, num_selected_t, ThreeWayPartition_CustomTypes_Fixture_Tag>(
+      less_op, greater_or_equal_op, input);
+  auto std_result =
+    std_partition(custom_less_than_t{comparison_value}, custom_greater_or_equal_t{comparison_value}, input);
+
+  REQUIRE(c_parallel_result == std_result);
+}
+
+struct ThreeWayPartition_Iterators_Fixture_Tag;
+C2H_TEST("ThreeWayPartition works with iterators", "[three_way_partition]")
+{
+  using key_t          = int;
+  using num_selected_t = int;
+
+  const std::size_t num_items    = GENERATE(0, 42, take(4, random(1 << 12, 1 << 20)));
+  const std::vector<key_t> input = generate<key_t>(num_items);
+  pointer_t<key_t> input_ptr(input);
+  pointer_t<key_t> first_part_output_ptr(num_items);
+  pointer_t<key_t> second_part_output_ptr(num_items);
+  pointer_t<key_t> unselected_output_ptr(num_items);
+  pointer_t<num_selected_t> num_selected_output_ptr(2);
+
+  iterator_t<key_t, random_access_iterator_state_t<key_t>> input_it =
+    make_random_access_iterator<key_t>(iterator_kind::INPUT, "int", "in");
+  input_it.state.data = input_ptr.ptr;
+
+  iterator_t<key_t, random_access_iterator_state_t<key_t>> first_part_output_it =
+    make_random_access_iterator<key_t>(iterator_kind::OUTPUT, "int", "first_part_output");
+  first_part_output_it.state.data = first_part_output_ptr.ptr;
+
+  iterator_t<key_t, random_access_iterator_state_t<key_t>> second_part_output_it =
+    make_random_access_iterator<key_t>(iterator_kind::OUTPUT, "int", "second_part_output");
+  second_part_output_it.state.data = second_part_output_ptr.ptr;
+
+  iterator_t<key_t, random_access_iterator_state_t<key_t>> unselected_output_it =
+    make_random_access_iterator<key_t>(iterator_kind::OUTPUT, "int", "unselected_output");
+  unselected_output_it.state.data = unselected_output_ptr.ptr;
+
+  iterator_t<key_t, random_access_iterator_state_t<key_t>> num_selected_output_it =
+    make_random_access_iterator<key_t>(iterator_kind::OUTPUT, "int", "num_selected_output");
+  num_selected_output_it.state.data = num_selected_output_ptr.ptr;
+
+  auto [less_op_src, greater_or_equal_op_src] = get_three_way_partition_ops(get_type_info<key_t>().type, 21);
+  operation_t less_op                         = make_operation("less_op", less_op_src);
+  operation_t greater_or_equal_op             = make_operation("greater_op", greater_or_equal_op_src);
+
+  auto& build_cache    = get_cache<ThreeWayPartition_Iterators_Fixture_Tag>();
+  const auto& test_key = make_key<key_t, num_selected_t>();
+
+  three_way_partition<false, true>(
+    input_it,
+    first_part_output_it,
+    second_part_output_it,
+    unselected_output_it,
+    num_selected_output_it,
+    less_op,
+    greater_or_equal_op,
+    num_items,
+    build_cache,
+    test_key);
+
+  std::vector<key_t> first_part_output(first_part_output_ptr);
+  std::vector<key_t> second_part_output(second_part_output_ptr);
+  std::vector<key_t> unselected_output(unselected_output_ptr);
+  std::vector<num_selected_t> num_selected(num_selected_output_ptr);
+
+  auto std_result = std_partition(less_than_t<key_t>{key_t{21}}, greater_or_equal_t<key_t>{key_t{21}}, input);
+
+  REQUIRE(first_part_output == std_result.first_part);
+  REQUIRE(second_part_output == std_result.second_part);
+  REQUIRE(unselected_output == std_result.unselected);
+  REQUIRE(static_cast<std::size_t>(num_selected[0]) == std_result.num_items_in_first_part);
+  REQUIRE(static_cast<std::size_t>(num_selected[1]) == std_result.num_items_in_second_part);
+  REQUIRE(num_items - static_cast<std::size_t>(num_selected[0] + num_selected[1]) == std_result.num_unselected_items);
+}
+
+#ifndef CCCL_C_PARALLEL_V2
+C2H_TEST("ThreeWayPartition build result has serialization metadata populated", "[three_way_partition][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  auto [first_src, second_src] = get_three_way_partition_ops(get_type_info<T>().type, 21);
+  operation_t first_op         = make_operation("less_op", first_src);
+  operation_t second_op        = make_operation("greater_op", second_src);
+
+  pointer_t<T> in(1);
+  pointer_t<T> first_out(1);
+  pointer_t<T> second_out(1);
+  pointer_t<T> unselected_out(1);
+  pointer_t<T> num_selected_out(1);
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_three_way_partition_build(
+      &build,
+      in,
+      first_out,
+      second_out,
+      unselected_out,
+      num_selected_out,
+      first_op,
+      second_op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path()));
+
+  CHECK(build.cc == build_info.get_cc_major() * 10 + build_info.get_cc_minor());
+  CHECK((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  CHECK(build.payload_size > 0);
+  CHECK(build.runtime_policy != nullptr);
+  CHECK(build.runtime_policy_size > 0);
+  REQUIRE(build.three_way_partition_init_kernel_lowered_name != nullptr);
+  CHECK(build.three_way_partition_init_kernel_lowered_name[0] != '\0');
+  REQUIRE(build.three_way_partition_kernel_lowered_name != nullptr);
+  CHECK(build.three_way_partition_kernel_lowered_name[0] != '\0');
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_three_way_partition_cleanup(&build));
+}
+
+C2H_TEST("ThreeWayPartition compile/load round-trip", "[three_way_partition][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+  constexpr T split_value = 21;
+
+  auto [first_src, second_src] = get_three_way_partition_ops(get_type_info<T>().type, split_value);
+  operation_t first_op         = make_operation("less_op", first_src);
+  operation_t second_op        = make_operation("greater_op", second_src);
+
+  pointer_t<T> dummy_in(1);
+  pointer_t<T> dummy_first_out(1);
+  pointer_t<T> dummy_second_out(1);
+  pointer_t<T> dummy_unselected_out(1);
+  pointer_t<T> dummy_num_selected_out(1);
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_three_way_partition_compile(
+      &build,
+      dummy_in,
+      dummy_first_out,
+      dummy_second_out,
+      dummy_unselected_out,
+      dummy_num_selected_out,
+      first_op,
+      second_op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+
+  REQUIRE((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  REQUIRE(build.payload_size > 0);
+  REQUIRE(build.three_way_partition_init_kernel_lowered_name != nullptr);
+  REQUIRE(build.three_way_partition_kernel_lowered_name != nullptr);
+  CHECK(build.library == nullptr);
+  CHECK(build.three_way_partition_init_kernel == nullptr);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_three_way_partition_load(&build));
+  REQUIRE(build.library != nullptr);
+  CHECK(build.three_way_partition_init_kernel != nullptr);
+  CHECK(build.three_way_partition_kernel != nullptr);
+
+  constexpr std::size_t n    = 16;
+  const std::vector<T> input = generate<T>(n);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> first_out(n);
+  pointer_t<T> second_out(n);
+  pointer_t<T> unselected_out(n);
+  pointer_t<int> num_selected_out(2);
+  CUstream null_stream      = nullptr;
+  size_t temp_storage_bytes = 0;
+
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_three_way_partition(
+      build,
+      nullptr,
+      &temp_storage_bytes,
+      input_ptr,
+      first_out,
+      second_out,
+      unselected_out,
+      num_selected_out,
+      first_op,
+      second_op,
+      n,
+      null_stream));
+  pointer_t<uint8_t> temp_storage(temp_storage_bytes);
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_three_way_partition(
+      build,
+      temp_storage.ptr,
+      &temp_storage_bytes,
+      input_ptr,
+      first_out,
+      second_out,
+      unselected_out,
+      num_selected_out,
+      first_op,
+      second_op,
+      n,
+      null_stream));
+
+  const int n_first      = num_selected_out[0];
+  const int n_second     = num_selected_out[1];
+  const int n_unselected = static_cast<int>(n) - n_first - n_second;
+  CHECK(n_first >= 0);
+  CHECK(n_second >= 0);
+  CHECK(n_unselected >= 0);
+  CHECK(n_first + n_second + n_unselected == static_cast<int>(n));
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_three_way_partition_cleanup(&build));
+}
+
+C2H_TEST("ThreeWayPartition compile rejects mismatched custom ops", "[three_way_partition][serialization]")
+{
+  using T                 = int32_t;
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+  constexpr T split_value = 21;
+
+  const auto [first_src, second_src] = get_three_way_partition_ops(get_type_info<T>().type, split_value);
+  operation_t regular_op             = make_operation("less_op", first_src);
+  (void) second_src;
+
+  // Kernel-only op: code_size == 0 with a non-empty name.
+  cccl_op_t custom_op{};
+  custom_op.type      = CCCL_STATELESS;
+  custom_op.name      = "greater_op";
+  custom_op.code_size = 0;
+  custom_op.code_type = CCCL_OP_LTOIR;
+  custom_op.size      = 1;
+  custom_op.alignment = 1;
+
+  pointer_t<T> dummy_in(1);
+  pointer_t<T> dummy_first_out(1);
+  pointer_t<T> dummy_second_out(1);
+  pointer_t<T> dummy_unselected_out(1);
+  pointer_t<T> dummy_num_selected_out(1);
+
+  cccl_device_three_way_partition_build_result_t build{};
+  // One regular op + one kernel-only op is an invalid combination.
+  REQUIRE(
+    CUDA_ERROR_INVALID_VALUE
+    == cccl_device_three_way_partition_compile(
+      &build,
+      dummy_in,
+      dummy_first_out,
+      dummy_second_out,
+      dummy_unselected_out,
+      dummy_num_selected_out,
+      regular_op,
+      custom_op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+}
+#endif // CCCL_C_PARALLEL_V2

cccl_upstream/c/parallel/test/test_transform.cpp

@@ -0,0 +1,987 @@
+#include <cstdint>
+#include <cstdlib>
+#include <numeric>
+#include <optional> // std::optional
+#include <string>
+
+#include <cuda_runtime.h>
+
+#include "algorithm_execution.h"
+#include "build_result_caching.h"
+#include "test_util.h"
+#include <cccl/c/transform.h>
+#include <cccl/c/types.h>
+
+using BuildResultT = cccl_device_transform_build_result_t;
+
+struct transform_cleanup
+{
+  CUresult operator()(BuildResultT* build_data) const noexcept
+  {
+    return cccl_device_transform_cleanup(build_data);
+  }
+};
+
+using transform_deleter       = BuildResultDeleter<BuildResultT, transform_cleanup>;
+using transform_build_cache_t = build_cache_t<std::string, result_wrapper_t<BuildResultT, transform_deleter>>;
+
+template <typename Tag>
+auto& get_cache()
+{
+  return fixture<transform_build_cache_t, Tag>::get_or_create().get_value();
+}
+
+struct transform_build
+{
+  using IterT = cccl_iterator_t;
+
+  template <typename... Ts>
+  CUresult operator()(BuildResultT* build_ptr, IterT input, IterT output, uint64_t, Ts... rest) const noexcept
+  {
+    return cccl_device_unary_transform_build(build_ptr, input, output, rest...);
+  }
+
+  template <typename... Ts>
+  CUresult
+  operator()(BuildResultT* build_ptr, IterT input1, IterT input2, IterT output, uint64_t, Ts... rest) const noexcept
+  {
+    return cccl_device_binary_transform_build(build_ptr, input1, input2, output, rest...);
+  }
+};
+
+struct unary_transform_run
+{
+  template <typename... Ts>
+  CUresult operator()(BuildResultT build, void* scratch, size_t* scratch_size, Ts... args) const noexcept
+  {
+    *scratch_size = 1;
+    return (scratch) ? cccl_device_unary_transform(build, args...) : CUDA_SUCCESS;
+  }
+};
+
+struct binary_transform_run
+{
+  template <typename... Ts>
+  CUresult operator()(BuildResultT build, void* scratch, size_t* scratch_size, Ts... args) const noexcept
+  {
+    *scratch_size = 1;
+    return (scratch) ? cccl_device_binary_transform(build, args...) : CUDA_SUCCESS;
+  }
+};
+
+template <typename BuildCache = transform_build_cache_t, typename KeyT = std::string>
+void unary_transform(
+  cccl_iterator_t input,
+  cccl_iterator_t output,
+  uint64_t num_items,
+  cccl_op_t op,
+  std::optional<BuildCache>& cache,
+  const std::optional<KeyT>& lookup_key)
+{
+  AlgorithmExecute<BuildResultT, transform_build, transform_cleanup, unary_transform_run, BuildCache, KeyT>(
+    cache, lookup_key, input, output, num_items, op);
+}
+
+template <typename BuildCache = transform_build_cache_t, typename KeyT = std::string>
+void binary_transform(
+  cccl_iterator_t input1,
+  cccl_iterator_t input2,
+  cccl_iterator_t output,
+  uint64_t num_items,
+  cccl_op_t op,
+  std::optional<BuildCache>& cache,
+  const std::optional<KeyT>& lookup_key)
+{
+  AlgorithmExecute<BuildResultT, transform_build, transform_cleanup, binary_transform_run, BuildCache, KeyT>(
+    cache, lookup_key, input1, input2, output, num_items, op);
+}
+
+C2H_TEST("Transform generates UBLKCP on SM90", "[transform][ublkcp]")
+{
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  // Only test for ublkcp when it is actually possible to get it.
+  if (build_info.get_cc_major() < 9)
+  {
+    return;
+  }
+
+  cccl_device_transform_build_result_t build{};
+  operation_t op = make_operation("op", get_unary_op(get_type_info<int>().type));
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_unary_transform_build(
+      &build,
+      pointer_t<int>(0),
+      pointer_t<int>(0),
+      op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path()));
+
+  std::string sass = inspect_sass(build.payload, build.payload_size);
+  CHECK(sass.find("UBLKCP") != std::string::npos);
+
+  op = make_operation("op", get_reduce_op(get_type_info<int>().type));
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_binary_transform_build(
+      &build,
+      pointer_t<int>(0),
+      pointer_t<int>(0),
+      pointer_t<int>(0),
+      op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path()));
+
+  sass = inspect_sass(build.payload, build.payload_size);
+  CHECK(sass.find("UBLKCP") != std::string::npos);
+}
+
+using integral_types = c2h::type_list<int32_t, uint32_t, int64_t, uint64_t>;
+struct Transform_IntegralTypes_Fixture_Tag;
+C2H_TEST("Transform works with integral types", "[transform]", integral_types)
+{
+  using T = c2h::get<0, TestType>;
+
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 16)));
+  operation_t op              = make_operation("op", get_unary_op(get_type_info<T>().type));
+  const std::vector<T> input  = generate<T>(num_items);
+  const std::vector<T> output(num_items, 0);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(output);
+
+  auto& build_cache    = get_cache<Transform_IntegralTypes_Fixture_Tag>();
+  const auto& test_key = make_key<T>();
+
+  unary_transform(input_ptr, output_ptr, num_items, op, build_cache, test_key);
+
+  std::vector<T> expected(num_items, 0);
+  std::transform(input.begin(), input.end(), expected.begin(), [](const T& x) {
+    return 2 * x;
+  });
+
+  if (num_items > 0)
+  {
+    REQUIRE(expected == std::vector<T>(output_ptr));
+  }
+}
+
+struct Transform_MisalignedInput_IntegerTypes_Fixture_Tag;
+C2H_TEST("Transform works with misaligned input with integral types", "[transform]", integral_types)
+{
+  using T = c2h::get<0, TestType>;
+
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 16)));
+  operation_t op              = make_operation("op", get_unary_op(get_type_info<T>().type));
+  const std::vector<T> input  = generate<T>(num_items + 1);
+  const std::vector<T> output(num_items, 0);
+  pointer_t<T> input_ptr_aligned(input);
+  pointer_t<T> input_ptr = input;
+  input_ptr.ptr += 1; // misalign by 1 from the guaranteed alignment of cudaMalloc, to maybe trip vectorized path
+  input_ptr.size -= 1;
+  pointer_t<T> output_ptr(output);
+
+  auto& build_cache    = get_cache<Transform_MisalignedInput_IntegerTypes_Fixture_Tag>();
+  const auto& test_key = make_key<T>();
+
+  unary_transform(input_ptr, output_ptr, num_items, op, build_cache, test_key);
+  input_ptr.ptr = nullptr; // avoid freeing the memory through this pointer
+
+  std::vector<T> expected(num_items, 0);
+  std::transform(input.begin() + 1, input.end(), expected.begin(), [](const T& x) {
+    return 2 * x;
+  });
+
+  REQUIRE(expected == std::vector<T>(output_ptr));
+}
+
+struct Transform_MisalignedOutput_IntegerTypes_Fixture_Tag;
+C2H_TEST("Transform works with misaligned output with integral types", "[transform]", integral_types)
+{
+  using T = c2h::get<0, TestType>;
+
+  const std::size_t num_items = GENERATE(1, 42, take(4, random(1 << 12, 1 << 16)));
+  operation_t op              = make_operation("op", get_unary_op(get_type_info<T>().type));
+  const std::vector<T> input  = generate<T>(num_items);
+  const std::vector<T> output(num_items + 1, 0);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr_aligned(output);
+  pointer_t<T> output_ptr = output;
+  output_ptr.ptr += 1; // misalign by 1 from the guaranteed alignment of cudaMalloc, to maybe trip vectorized path
+  output_ptr.size -= 1;
+
+  auto& build_cache    = get_cache<Transform_MisalignedOutput_IntegerTypes_Fixture_Tag>();
+  const auto& test_key = make_key<T>();
+
+  unary_transform(input_ptr, output_ptr, num_items, op, build_cache, test_key);
+
+  std::vector<T> expected(num_items, 0);
+  std::transform(input.begin(), input.end(), expected.begin(), [](const T& x) {
+    return 2 * x;
+  });
+
+  REQUIRE(expected == std::vector<T>(output_ptr));
+
+  output_ptr.ptr = nullptr; // avoid freeing the memory through this pointer
+}
+
+struct Transform_IntegralTypes_WellKnown_Fixture_Tag;
+C2H_TEST("Transform works with integral types with well-known operations", "[transform][well_known]", integral_types)
+{
+  using T = c2h::get<0, TestType>;
+
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 16)));
+  cccl_op_t op                = make_well_known_unary_operation();
+  const std::vector<T> input  = generate<T>(num_items);
+  const std::vector<T> output(num_items, 0);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(output);
+
+  auto& build_cache    = get_cache<Transform_IntegralTypes_WellKnown_Fixture_Tag>();
+  const auto& test_key = make_key<T>();
+
+  unary_transform(input_ptr, output_ptr, num_items, op, build_cache, test_key);
+
+  std::vector<T> expected(num_items, 0);
+  _CCCL_DIAG_PUSH
+  _CCCL_DIAG_SUPPRESS_MSVC(4146) // unary minus on unsigned type
+  std::transform(input.begin(), input.end(), expected.begin(), [](const T& x) {
+    return -x;
+  });
+  _CCCL_DIAG_POP
+  if (num_items > 0)
+  {
+    REQUIRE(expected == std::vector<T>(output_ptr));
+  }
+}
+
+C2H_TEST("Transform works with logical and bitwise well-known operations", "[transform][well_known]")
+{
+  std::optional<transform_build_cache_t> no_cache = std::nullopt;
+  const std::optional<std::string> no_key         = std::nullopt;
+
+  {
+    const std::vector<uint32_t> input{0U, 1U, 0xaaaaaaaaU, 0xffffffffU};
+    pointer_t<uint32_t> input_ptr(input);
+    pointer_t<uint32_t> output_ptr(input.size());
+
+    cccl_op_t bit_not_op = make_well_known_unary_operation();
+    bit_not_op.type      = cccl_op_kind_t::CCCL_BIT_NOT;
+    unary_transform(input_ptr, output_ptr, input.size(), bit_not_op, no_cache, no_key);
+
+    REQUIRE(std::vector<uint32_t>(output_ptr) == std::vector<uint32_t>{0xffffffffU, 0xfffffffeU, 0x55555555U, 0U});
+  }
+
+  const std::vector<uint8_t> lhs{1, 1, 0, 0};
+  const std::vector<uint8_t> rhs{1, 0, 1, 0};
+  pointer_t<uint8_t> lhs_ptr(lhs);
+  pointer_t<uint8_t> rhs_ptr(rhs);
+
+  const auto check_logical_op = [&](cccl_op_kind_t kind, const std::vector<uint8_t>& expected) {
+    pointer_t<uint8_t> output_ptr(lhs.size());
+    cccl_op_t op = make_well_known_binary_operation();
+    op.type      = kind;
+
+    binary_transform(
+      make_boolean_iterator(lhs_ptr),
+      make_boolean_iterator(rhs_ptr),
+      make_boolean_iterator(output_ptr),
+      lhs.size(),
+      op,
+      no_cache,
+      no_key);
+
+    REQUIRE(std::vector<uint8_t>(output_ptr) == expected);
+  };
+
+  check_logical_op(cccl_op_kind_t::CCCL_LOGICAL_AND, {1, 0, 0, 0});
+  check_logical_op(cccl_op_kind_t::CCCL_LOGICAL_OR, {1, 1, 1, 0});
+}
+
+struct pair
+{
+  short a;
+  size_t b;
+
+  bool operator==(const pair& other) const
+  {
+    return a == other.a && b == other.b;
+  }
+};
+
+struct custom_int
+{
+  int value;
+};
+
+C2H_TEST("Transform works with C++ source for custom types with a well-known unary operation",
+         "[transform][well_known][cpp_source]")
+{
+  const std::string source = R"(
+struct custom_int { int value; };
+extern "C" __device__ void logical_not_custom_int(void* input_ptr, void* output_ptr) {
+  const custom_int* input = static_cast<const custom_int*>(input_ptr);
+  bool* output = static_cast<bool*>(output_ptr);
+  *output = !input->value;
+}
+)";
+  const std::vector<custom_int> input{{0}, {1}, {-2}, {42}};
+  pointer_t<custom_int> input_ptr(input);
+  std::optional<transform_build_cache_t> no_cache = std::nullopt;
+  const std::optional<std::string> no_key         = std::nullopt;
+
+  operation_t op_state = make_cpp_operation("logical_not_custom_int", source);
+  cccl_op_t op         = op_state;
+  op.type              = cccl_op_kind_t::CCCL_LOGICAL_NOT;
+  pointer_t<uint8_t> output_ptr(input.size());
+
+  unary_transform(input_ptr, make_boolean_iterator(output_ptr), input.size(), op, no_cache, no_key);
+
+  REQUIRE(std::vector<uint8_t>(output_ptr) == std::vector<uint8_t>{1, 0, 0, 0});
+}
+
+struct Transform_DifferentOutputTypes_Fixture_Tag;
+C2H_TEST("Transform works with output of different type", "[transform]")
+{
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 24)));
+
+  operation_t op               = make_operation("op",
+                                  R"(struct pair { short a; size_t b; };
+extern "C" __device__ void op(void* x_ptr, void* out_ptr) {
+  int* x = static_cast<int*>(x_ptr);
+  pair* out = static_cast<pair*>(out_ptr);
+  *out = pair{ short(*x), size_t(*x) };
+})");
+  const std::vector<int> input = generate<int>(num_items);
+  std::vector<pair> expected(num_items);
+  std::vector<pair> output(num_items);
+  for (std::size_t i = 0; i < num_items; ++i)
+  {
+    expected[i] = {short(input[i]), size_t(input[i])};
+  }
+  pointer_t<int> input_ptr(input);
+  pointer_t<pair> output_ptr(output);
+
+  auto& build_cache    = get_cache<Transform_DifferentOutputTypes_Fixture_Tag>();
+  const auto& test_key = make_key<int, pair>();
+
+  unary_transform(input_ptr, output_ptr, num_items, op, build_cache, test_key);
+
+  if (num_items > 0)
+  {
+    REQUIRE(expected == std::vector<pair>(output_ptr));
+  }
+}
+
+struct alignas(8) unary_storage_in
+{
+  int x;
+  short y;
+};
+
+struct alignas(16) unary_storage_out
+{
+  long long sum;
+  int diff;
+
+  bool operator==(const unary_storage_out& other) const
+  {
+    return sum == other.sum && diff == other.diff;
+  }
+};
+
+struct Transform_UnaryStorageTypes_Fixture_Tag;
+C2H_TEST("Transform works with unary storage types of different size/alignment", "[transform]")
+{
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 16)));
+
+  operation_t op = make_operation("op",
+                                  R"(struct alignas(8) unary_storage_in { int x; short y; };
+struct alignas(16) unary_storage_out { long long sum; int diff; };
+extern "C" __device__ void op(void* x_ptr, void* out_ptr) {
+  auto* x = static_cast<unary_storage_in*>(x_ptr);
+  auto* out = static_cast<unary_storage_out*>(out_ptr);
+  out->sum = static_cast<long long>(x->x) + x->y;
+  out->diff = x->x - x->y;
+})");
+
+  std::vector<unary_storage_in> input(num_items);
+  std::vector<unary_storage_out> output(num_items);
+  std::vector<unary_storage_out> expected(num_items);
+  for (std::size_t i = 0; i < num_items; ++i)
+  {
+    input[i]    = {static_cast<int>(i + 3), static_cast<short>(i % 7)};
+    expected[i] = {static_cast<long long>(input[i].x) + input[i].y, input[i].x - input[i].y};
+  }
+
+  pointer_t<unary_storage_in> input_ptr(input);
+  pointer_t<unary_storage_out> output_ptr(output);
+
+  auto& build_cache    = get_cache<Transform_UnaryStorageTypes_Fixture_Tag>();
+  const auto& test_key = make_key<unary_storage_in, unary_storage_out>();
+
+  unary_transform(input_ptr, output_ptr, num_items, op, build_cache, test_key);
+
+  if (num_items > 0)
+  {
+    REQUIRE(expected == std::vector<unary_storage_out>(output_ptr));
+  }
+}
+
+struct Transform_CustomTypes_Fixture_Tag;
+C2H_TEST("Transform works with custom types", "[transform]")
+{
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 24)));
+
+  operation_t op              = make_operation("op",
+                                  R"(struct pair { short a; size_t b; };
+extern "C" __device__ void op(void* x_ptr, void* out_ptr) {
+  pair* x = static_cast<pair*>(x_ptr);
+  pair* out = static_cast<pair*>(out_ptr);
+  *out = pair{ x->a * 2, x->b * 2  };
+})");
+  const std::vector<short> a  = generate<short>(num_items);
+  const std::vector<size_t> b = generate<size_t>(num_items);
+  std::vector<pair> input(num_items);
+  std::vector<pair> output(num_items);
+  for (std::size_t i = 0; i < num_items; ++i)
+  {
+    input[i] = pair{a[i], b[i]};
+  }
+  pointer_t<pair> input_ptr(input);
+  pointer_t<pair> output_ptr(output);
+
+  auto& build_cache    = get_cache<Transform_CustomTypes_Fixture_Tag>();
+  const auto& test_key = make_key<pair, pair>();
+
+  unary_transform(input_ptr, output_ptr, num_items, op, build_cache, test_key);
+
+  std::vector<pair> expected(num_items, {0, 0});
+  std::transform(input.begin(), input.end(), expected.begin(), [](const pair& x) {
+    return pair{short(x.a * 2), x.b * 2};
+  });
+  if (num_items > 0)
+  {
+    REQUIRE(expected == std::vector<pair>(output_ptr));
+  }
+}
+
+struct Transform_CustomTypes_WellKnown_Fixture_Tag;
+C2H_TEST("Transform works with custom types with well-known operators", "[transform][well_known]")
+{
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 24)));
+
+  operation_t op_state = make_operation("op",
+                                        R"(struct pair { short a; size_t b; };
+extern "C" __device__ void op(void* x_ptr, void* out_ptr) {
+  pair* x = static_cast<pair*>(x_ptr);
+  pair* out = static_cast<pair*>(out_ptr);
+  *out = pair{ x->a * 2, x->b * 2  };
+})");
+  cccl_op_t op         = op_state;
+  // HACK: this doesn't actually match the operation above, but that's fine, as we are supposed to not take the
+  // well-known path anyway
+  op.type                     = cccl_op_kind_t::CCCL_NEGATE;
+  const std::vector<short> a  = generate<short>(num_items);
+  const std::vector<size_t> b = generate<size_t>(num_items);
+  std::vector<pair> input(num_items);
+  std::vector<pair> output(num_items);
+  for (std::size_t i = 0; i < num_items; ++i)
+  {
+    input[i] = pair{a[i], b[i]};
+  }
+  pointer_t<pair> input_ptr(input);
+  pointer_t<pair> output_ptr(output);
+
+  auto& build_cache    = get_cache<Transform_CustomTypes_WellKnown_Fixture_Tag>();
+  const auto& test_key = make_key<pair, pair>();
+
+  unary_transform(input_ptr, output_ptr, num_items, op, build_cache, test_key);
+
+  std::vector<pair> expected(num_items, {0, 0});
+  std::transform(input.begin(), input.end(), expected.begin(), [](const pair& x) {
+    return pair{short(x.a * 2), x.b * 2};
+  });
+  if (num_items > 0)
+  {
+    REQUIRE(expected == std::vector<pair>(output_ptr));
+  }
+}
+
+struct Transform_InputIterators_Fixture_Tag;
+C2H_TEST("Transform works with input iterators", "[transform]")
+{
+  const std::size_t num_items = GENERATE(1, 42, take(1, random(1 << 12, 1 << 16)));
+  operation_t op              = make_operation("op", get_unary_op(get_type_info<int>().type));
+  iterator_t<int, counting_iterator_state_t<int>> input_it = make_counting_iterator<int>("int");
+  input_it.state.value                                     = 0;
+  pointer_t<int> output_it(num_items);
+
+  auto& build_cache    = get_cache<Transform_InputIterators_Fixture_Tag>();
+  const auto& test_key = make_key<int>();
+
+  unary_transform(input_it, output_it, num_items, op, build_cache, test_key);
+
+  // vector storing a sequence of values 0, 1, 2, ..., num_items - 1
+  std::vector<int> input(num_items);
+  std::iota(input.begin(), input.end(), 0);
+
+  std::vector<int> expected(num_items);
+  std::transform(input.begin(), input.end(), expected.begin(), [](const int& x) {
+    return x * 2;
+  });
+  if (num_items > 0)
+  {
+    REQUIRE(expected == std::vector<int>(output_it));
+  }
+}
+
+struct Transform_OutputIterators_Fixture_Tag;
+C2H_TEST("Transform works with output iterators", "[transform]")
+{
+  const int num_items = GENERATE(1, 42, take(1, random(1 << 12, 1 << 16)));
+  operation_t op      = make_operation("op", get_unary_op(get_type_info<int>().type));
+  iterator_t<int, random_access_iterator_state_t<int>> output_it =
+    make_random_access_iterator<int>(iterator_kind::OUTPUT, "int", "out", " * 2");
+  const std::vector<int> input = generate<int>(num_items);
+  pointer_t<int> input_it(input);
+  pointer_t<int> inner_output_it(num_items);
+  output_it.state.data = inner_output_it.ptr;
+
+  auto& build_cache    = get_cache<Transform_OutputIterators_Fixture_Tag>();
+  const auto& test_key = make_key<int>();
+
+  unary_transform(input_it, output_it, num_items, op, build_cache, test_key);
+
+  std::vector<int> expected(num_items);
+  std::transform(input.begin(), input.end(), expected.begin(), [](int x) {
+    return x * 4;
+  });
+  if (num_items > 0)
+  {
+    REQUIRE(expected == std::vector<int>(inner_output_it));
+  }
+}
+
+struct Transform_BinaryOp_Fixture_Tag;
+C2H_TEST("Transform with binary operator", "[transform]")
+{
+  const std::size_t num_items   = GENERATE(0, 42, take(4, random(1 << 12, 1 << 16)));
+  const std::vector<int> input1 = generate<int>(num_items);
+  const std::vector<int> input2 = generate<int>(num_items);
+  const std::vector<int> output(num_items, 0);
+  pointer_t<int> input1_ptr(input1);
+  pointer_t<int> input2_ptr(input2);
+  pointer_t<int> output_ptr(output);
+
+  operation_t op = make_operation("op",
+                                  R"(extern "C" __device__ void op(void* x_ptr, void* y_ptr, void* out_ptr  ) {
+  int* x = static_cast<int*>(x_ptr);
+  int* y = static_cast<int*>(y_ptr);
+  int* out = static_cast<int*>(out_ptr);
+  *out = (*x > *y) ? *x : *y;
+})");
+
+  auto& build_cache    = get_cache<Transform_BinaryOp_Fixture_Tag>();
+  const auto& test_key = make_key<int>();
+
+  binary_transform(input1_ptr, input2_ptr, output_ptr, num_items, op, build_cache, test_key);
+
+  std::vector<int> expected(num_items, 0);
+  std::transform(input1.begin(), input1.end(), input2.begin(), expected.begin(), [](const int& x, const int& y) {
+    return (x > y) ? x : y;
+  });
+
+  if (num_items > 0)
+  {
+    REQUIRE(expected == std::vector<int>(output_ptr));
+  }
+}
+
+struct alignas(16) binary_storage_in1
+{
+  long long a;
+  int b;
+};
+
+struct alignas(8) binary_storage_in2
+{
+  int c;
+  int d;
+};
+
+struct alignas(16) binary_storage_out
+{
+  long long sum;
+  int diff;
+
+  bool operator==(const binary_storage_out& other) const
+  {
+    return sum == other.sum && diff == other.diff;
+  }
+};
+
+struct Transform_BinaryStorageTypes_Fixture_Tag;
+C2H_TEST("Transform works with binary storage types of different size/alignment", "[transform]")
+{
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 16)));
+
+  operation_t op = make_operation("op",
+                                  R"(struct alignas(16) binary_storage_in1 { long long a; int b; };
+struct alignas(8) binary_storage_in2 { int c; int d; };
+struct alignas(16) binary_storage_out { long long sum; int diff; };
+extern "C" __device__ void op(void* x_ptr, void* y_ptr, void* out_ptr) {
+  auto* x = static_cast<binary_storage_in1*>(x_ptr);
+  auto* y = static_cast<binary_storage_in2*>(y_ptr);
+  auto* out = static_cast<binary_storage_out*>(out_ptr);
+  out->sum = x->a + static_cast<long long>(y->c);
+  out->diff = x->b - y->d;
+})");
+
+  std::vector<binary_storage_in1> input1(num_items);
+  std::vector<binary_storage_in2> input2(num_items);
+  std::vector<binary_storage_out> output(num_items);
+  std::vector<binary_storage_out> expected(num_items);
+  for (std::size_t i = 0; i < num_items; ++i)
+  {
+    input1[i]   = {static_cast<long long>(i + 5), static_cast<int>(i + 2)};
+    input2[i]   = {static_cast<int>(i + 7), static_cast<int>(i + 1)};
+    expected[i] = {input1[i].a + static_cast<long long>(input2[i].c), input1[i].b - input2[i].d};
+  }
+
+  pointer_t<binary_storage_in1> input1_ptr(input1);
+  pointer_t<binary_storage_in2> input2_ptr(input2);
+  pointer_t<binary_storage_out> output_ptr(output);
+
+  auto& build_cache    = get_cache<Transform_BinaryStorageTypes_Fixture_Tag>();
+  const auto& test_key = make_key<binary_storage_in1, binary_storage_in2, binary_storage_out>();
+
+  binary_transform(input1_ptr, input2_ptr, output_ptr, num_items, op, build_cache, test_key);
+
+  if (num_items > 0)
+  {
+    REQUIRE(expected == std::vector<binary_storage_out>(output_ptr));
+  }
+}
+
+struct Transform_BinaryOp_Iterator_Fixture_Tag;
+C2H_TEST("Binary transform with one iterator", "[transform]")
+{
+  const std::size_t num_items   = GENERATE(0, 42, take(4, random(1 << 12, 1 << 16)));
+  const std::vector<int> input1 = generate<int>(num_items);
+
+  iterator_t<int, counting_iterator_state_t<int>> input2_it = make_counting_iterator<int>("int");
+  input2_it.state.value                                     = 0;
+
+  const std::vector<int> output(num_items, 0);
+  pointer_t<int> input1_ptr(input1);
+  pointer_t<int> output_ptr(output);
+
+  operation_t op = make_operation("op",
+                                  R"(extern "C" __device__ void op(void* x_ptr, void* y_ptr, void* out_ptr) {
+  int* x = static_cast<int*>(x_ptr);
+  int* y = static_cast<int*>(y_ptr);
+  int* out = static_cast<int*>(out_ptr);
+  *out = (*x > *y) ? *x : *y;
+})");
+
+  auto& build_cache    = get_cache<Transform_BinaryOp_Iterator_Fixture_Tag>();
+  const auto& test_key = make_key<int>();
+
+  binary_transform(input1_ptr, input2_it, output_ptr, num_items, op, build_cache, test_key);
+
+  std::vector<int> input2(num_items);
+  std::iota(input2.begin(), input2.end(), 0);
+  std::vector<int> expected(num_items, 0);
+  std::transform(input1.begin(), input1.end(), input2.begin(), expected.begin(), [](const int& x, const int& y) {
+    return (x > y) ? x : y;
+  });
+
+  if (num_items > 0)
+  {
+    REQUIRE(expected == std::vector<int>(output_ptr));
+  }
+}
+
+using floating_point_types = c2h::type_list<
+#if _CCCL_HAS_NVFP16()
+  __half,
+#endif
+  float,
+  double>;
+struct Transform_FloatingPointTypes_Fixture_Tag;
+C2H_TEST("Transform works with floating point types", "[transform]", floating_point_types)
+{
+  using T = c2h::get<0, TestType>;
+
+  const std::size_t num_items      = GENERATE(0, 42, take(4, random(1 << 12, 1 << 16)));
+  operation_t op                   = make_operation("op", get_unary_op(get_type_info<T>().type));
+  const std::vector<int> int_input = generate<int>(num_items);
+  // Suppress harmless conversion warnings on MSVC
+  _CCCL_DIAG_PUSH
+  _CCCL_DIAG_SUPPRESS_MSVC(4244)
+  const std::vector<T> input(int_input.begin(), int_input.end());
+  _CCCL_DIAG_POP
+  const std::vector<T> output(num_items, 0);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(output);
+
+  auto& build_cache    = get_cache<Transform_FloatingPointTypes_Fixture_Tag>();
+  const auto& test_key = make_key<T>();
+
+  unary_transform(input_ptr, output_ptr, num_items, op, build_cache, test_key);
+
+  std::vector<T> expected(num_items, 0);
+  std::transform(input.begin(), input.end(), expected.begin(), [](const T& x) {
+    return T{2} * x;
+  });
+
+  if (num_items > 0)
+  {
+    REQUIRE(expected == std::vector<T>(output_ptr));
+  }
+}
+
+C2H_TEST("Transform works with C++ source operations", "[transform]")
+{
+  using T = int32_t;
+
+  const std::size_t num_items = GENERATE(42, 1337, 42000);
+
+  // Create operation from C++ source instead of LTO-IR
+  std::string cpp_source = R"(
+    extern "C" __device__ void op(void* input, void* output) {
+      int* in = (int*)input;
+      int* out = (int*)output;
+      *out = *in * 2;
+    }
+  )";
+
+  operation_t op = make_cpp_operation("op", cpp_source);
+
+  const std::vector<T> input = generate<T>(num_items);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(num_items);
+
+  // Test key including flag that this uses C++ source
+  std::optional<std::string> test_key = std::format("cpp_source_test_{}_{}", num_items, typeid(T).name());
+
+  auto& cache = fixture<transform_build_cache_t, Transform_IntegralTypes_Fixture_Tag>::get_or_create().get_value();
+  std::optional<transform_build_cache_t> cache_opt = cache;
+
+  unary_transform(input_ptr, output_ptr, num_items, op, cache_opt, test_key);
+
+  const std::vector<T> output = output_ptr;
+  std::vector<T> expected     = input;
+  std::transform(expected.begin(), expected.end(), expected.begin(), [](T x) {
+    return x * 2;
+  });
+  REQUIRE(output == expected);
+}
+
+C2H_TEST("Transform works with C++ source operations using custom headers", "[transform]")
+{
+  using T = int32_t;
+
+  const std::size_t num_items = GENERATE(42, 1337, 42000);
+
+  // Create operation from C++ source that uses the identity function from header
+  std::string cpp_source = R"(
+    #include "test_identity.h"
+    extern "C" __device__ void op(void* input, void* output) {
+      int* in = (int*)input;
+      int* out = (int*)output;
+      int val = test_identity(*in);
+      *out = val * 2;
+    }
+  )";
+
+  operation_t op = make_cpp_operation("op", cpp_source);
+
+  const std::vector<T> input = generate<T>(num_items);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(num_items);
+
+  // Test _ex version with custom build configuration
+  const char* extra_flags[] = {"-DTEST_IDENTITY_ENABLED"};
+  const char* extra_dirs[]  = {TEST_INCLUDE_PATH};
+  cccl_build_config config  = make_build_config(extra_flags, 1, extra_dirs, 1);
+
+  // Build with _ex version
+  cccl_device_transform_build_result_t build{};
+  const auto& build_info = BuildInformation<>::init();
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_unary_transform_build_ex(
+      &build,
+      input_ptr,
+      output_ptr,
+      op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      &config));
+
+  // Execute the transform
+  REQUIRE(CUDA_SUCCESS == cccl_device_unary_transform(build, input_ptr, output_ptr, num_items, op, CU_STREAM_LEGACY));
+
+  // Verify results
+  std::vector<T> output(num_items);
+  cudaMemcpy(output.data(), static_cast<void*>(output_ptr.ptr), sizeof(T) * num_items, cudaMemcpyDeviceToHost);
+  std::vector<T> expected = input;
+  std::transform(expected.begin(), expected.end(), expected.begin(), [](T x) {
+    return x * 2;
+  });
+  REQUIRE(output == expected);
+
+  // Cleanup
+  REQUIRE(CUDA_SUCCESS == cccl_device_transform_cleanup(&build));
+}
+
+struct transform_stateful_counter_state_t
+{
+  int* d_counter;
+};
+
+C2H_TEST("Transform works with stateful unary operators", "[transform]")
+{
+  const std::size_t num_items = GENERATE(0, 42, take(4, random(1 << 12, 1 << 16)));
+  const std::vector<int> host_counter{0};
+  pointer_t<int> counter(host_counter);
+  stateful_operation_t<transform_stateful_counter_state_t> op = make_operation(
+    "op",
+    R"(struct transform_stateful_counter_state_t { int* d_counter; };
+extern "C" __device__ void op(void* state_ptr, void* x_ptr, void* out_ptr) {
+  auto* state = static_cast<transform_stateful_counter_state_t*>(state_ptr);
+  atomicAdd(state->d_counter, 1);
+  int x = *static_cast<int*>(x_ptr);
+  *static_cast<int*>(out_ptr) = x * 2;
+})",
+    transform_stateful_counter_state_t{counter.ptr});
+
+  const std::vector<int> input = generate<int>(num_items);
+  const std::vector<int> output(num_items, 0);
+  pointer_t<int> input_ptr(input);
+  pointer_t<int> output_ptr(output);
+
+  std::optional<transform_build_cache_t> build_cache = std::nullopt;
+  std::optional<std::string> test_key                = std::nullopt;
+
+  unary_transform(input_ptr, output_ptr, num_items, op, build_cache, test_key);
+
+  std::vector<int> expected(num_items, 0);
+  std::transform(input.begin(), input.end(), expected.begin(), [](int x) {
+    return x * 2;
+  });
+
+  if (num_items > 0)
+  {
+    REQUIRE(expected == std::vector<int>(output_ptr));
+    REQUIRE(counter[0] == static_cast<int>(num_items));
+  }
+}
+
+#ifndef CCCL_C_PARALLEL_V2
+C2H_TEST("Transform build result has serialization metadata populated", "[transform][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  cccl_op_t op = make_well_known_unary_operation();
+  pointer_t<T> in(1);
+  pointer_t<T> out(1);
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_unary_transform_build(
+      &build,
+      in,
+      out,
+      op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path()));
+
+  CHECK(build.cc == build_info.get_cc_major() * 10 + build_info.get_cc_minor());
+  CHECK((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  CHECK(build.payload_size > 0);
+  CHECK(build.runtime_policy != nullptr);
+  CHECK(build.runtime_policy_size > 0);
+  REQUIRE(build.transform_kernel_lowered_name != nullptr);
+  CHECK(build.transform_kernel_lowered_name[0] != '\0');
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_transform_cleanup(&build));
+}
+
+C2H_TEST("Transform compile/load round-trip", "[transform][serialization]")
+{
+  using T = int32_t;
+
+  constexpr int device_id = 0;
+  const auto& build_info  = BuildInformation<device_id>::init();
+
+  cccl_op_t op = make_well_known_unary_operation();
+  pointer_t<T> dummy_in(1);
+  pointer_t<T> dummy_out(1);
+
+  BuildResultT build{};
+  REQUIRE(
+    CUDA_SUCCESS
+    == cccl_device_unary_transform_compile(
+      &build,
+      dummy_in,
+      dummy_out,
+      op,
+      build_info.get_cc_major(),
+      build_info.get_cc_minor(),
+      build_info.get_cub_path(),
+      build_info.get_thrust_path(),
+      build_info.get_libcudacxx_path(),
+      build_info.get_ctk_path(),
+      nullptr));
+
+  REQUIRE((build.payload != nullptr && build.payload_kind == CCCL_PAYLOAD_CUBIN));
+  REQUIRE(build.payload_size > 0);
+  REQUIRE(build.transform_kernel_lowered_name != nullptr);
+  CHECK(build.library == nullptr);
+  CHECK(build.transform_kernel == nullptr);
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_transform_load(&build));
+  REQUIRE(build.library != nullptr);
+  CHECK(build.transform_kernel != nullptr);
+
+  constexpr std::size_t n    = 16;
+  const std::vector<T> input = generate<T>(n);
+  pointer_t<T> input_ptr(input);
+  pointer_t<T> output_ptr(n);
+  CUstream null_stream = nullptr;
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_unary_transform(build, input_ptr, output_ptr, n, op, null_stream));
+
+  std::vector<T> expected(input);
+  std::transform(expected.begin(), expected.end(), expected.begin(), [](T x) {
+    return -x;
+  });
+  REQUIRE(expected == std::vector<T>(output_ptr));
+
+  REQUIRE(CUDA_SUCCESS == cccl_device_transform_cleanup(&build));
+}
+#endif // CCCL_C_PARALLEL_V2