[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/c/parallel/src/util/build_utils.h

@@ -0,0 +1,46 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA Core Compute Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <vector>
+
+#include <cccl/c/types.h>
+
+namespace cccl::detail
+{
+/**
+ * @brief Extends a vector of compilation arguments with extra flags and include directories from a build config
+ *
+ * @param args The vector of arguments to extend
+ * @param config The build configuration containing extra flags and include directories (can be nullptr)
+ */
+inline void extend_args_with_build_config(std::vector<const char*>& args, const cccl_build_config* config)
+{
+  // Work around an NVRTC 13.3 diagnostic pragma bug in CUDA Toolkit headers that leaves deprecated vector type warnings
+  // unsuppressed when including headers such as cuda_fp8.h, cuda_fp6.h, and cuda_fp4.h.
+  args.push_back("-D__NV_NO_VECTOR_DEPRECATION_DIAG");
+
+  if (config)
+  {
+    // Add extra compile flags
+    for (size_t i = 0; i < config->num_extra_compile_flags; ++i)
+    {
+      args.push_back(config->extra_compile_flags[i]);
+    }
+    // Add include directories
+    for (size_t i = 0; i < config->num_extra_include_dirs; ++i)
+    {
+      args.push_back("-I");
+      args.push_back(config->extra_include_dirs[i]);
+    }
+  }
+}
+} // namespace cccl::detail

cccl_upstream/c/parallel/src/util/context.cpp

@@ -0,0 +1,31 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <stdexcept>
+
+#include "errors.h"
+
+bool try_push_context()
+{
+  CUcontext context = nullptr;
+
+  check(cuCtxGetCurrent(&context));
+
+  if (context == nullptr)
+  {
+    const int default_device = 0;
+    check(cuDevicePrimaryCtxRetain(&context, default_device));
+    check(cuCtxPushCurrent(context));
+
+    return true;
+  }
+
+  return false;
+}

cccl_upstream/c/parallel/src/util/context.h

@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <cuda.h>
+#include <nvrtc.h>
+
+#include <nvrtc/nvjitlink_helper.h>
+
+bool try_push_context();

cccl_upstream/c/parallel/src/util/errors.cpp

@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include "errors.h"
+
+#include <stdexcept>
+
+void check(nvrtcResult result)
+{
+  if (result != NVRTC_SUCCESS)
+  {
+    throw std::runtime_error(std::string("NVRTC error: ") + nvrtcGetErrorString(result));
+  }
+}
+
+void check(CUresult result)
+{
+  if (result != CUDA_SUCCESS)
+  {
+    const char* str = nullptr;
+    cuGetErrorString(result, &str);
+    throw std::runtime_error(std::string("CUDA error: ") + str);
+  }
+}
+
+void check(nvJitLinkResult result)
+{
+  if (result != NVJITLINK_SUCCESS)
+  {
+    throw std::runtime_error(std::string("nvJitLink error: ") + std::to_string(result));
+  }
+}

cccl_upstream/c/parallel/src/util/errors.h

@@ -0,0 +1,20 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <cuda.h>
+#include <nvrtc.h>
+
+#include <nvrtc/nvjitlink_helper.h>
+
+void check(nvrtcResult result);
+void check(CUresult result);
+void check(nvJitLinkResult result);

cccl_upstream/c/parallel/src/util/indirect_arg.h

@@ -0,0 +1,119 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <cstdint>
+#include <stdexcept>
+#include <type_traits>
+
+#include <cccl/c/types.h>
+
+// GCC emits a dangling pointer warning in the `LargeSegmentsSelector` and
+// `SmallSegmentsSelector` functions in segmented_sort.cu. The warning occurs
+// when `return indirect_arg_t()` invokes the `indirect_arg_t` constructor that
+// accepts a `cccl_op_t`. Even though this is a stateless op, we must initialize
+// the `ptr` member to a valid address.
+//
+// We cannot use `nullptr` because the pointer is passed to a driver API that
+// requires the size of empty arguments to be 1 (not 0), meaning it will attempt
+// to copy a byte from the address.
+//
+// Initially, we initialized `ptr` to `this`, but this triggered the dangling
+// pointer warning in GCC. To avoid this, we use a global variable instead.
+static inline char _global_storage = 0;
+
+struct indirect_arg_t
+{
+  void* ptr;
+
+  indirect_arg_t(cccl_iterator_t& it)
+      : ptr(it.type == cccl_iterator_kind_t::CCCL_POINTER ? &it.state : it.state)
+  {}
+
+  indirect_arg_t(cccl_op_t& op)
+      : ptr(op.type == cccl_op_kind_t::CCCL_STATEFUL ? op.state : &_global_storage)
+  {}
+
+  indirect_arg_t(cccl_value_t& val)
+      : ptr(val.state)
+  {}
+
+  void* operator&() const
+  {
+    return ptr;
+  }
+};
+
+template <typename U>
+concept Increment64 = std::is_integral_v<U> && sizeof(U) == sizeof(int64_t);
+
+struct indirect_iterator_t
+{
+  void* ptr;
+  size_t value_size;
+  cccl_host_op_fn_ptr_t host_advance_fn_p;
+
+  indirect_iterator_t(cccl_iterator_t& it)
+      : ptr{nullptr}
+      , value_size{0}
+      , host_advance_fn_p{nullptr}
+  {
+    if (it.type == cccl_iterator_kind_t::CCCL_POINTER)
+    {
+      value_size = it.value_type.size;
+      ptr        = &it.state;
+    }
+    else
+    {
+      ptr               = it.state;
+      host_advance_fn_p = it.host_advance;
+    }
+  }
+
+  void* operator&() const
+  {
+    return ptr;
+  }
+
+  template <Increment64 U>
+  void operator+=(U offset)
+  {
+    if (value_size)
+    {
+      // CCCL_POINTER case
+      // ptr is a pointer to pointer we need to increment
+      // read the iterator pointer value
+      char*& p = *static_cast<char**>(ptr);
+      // increment the value
+      p += (offset * value_size);
+    }
+    else
+    {
+      if (host_advance_fn_p)
+      {
+        if constexpr (std::is_signed_v<U>)
+        {
+          cccl_increment_t incr{.signed_offset = offset};
+          (*host_advance_fn_p)(ptr, incr);
+        }
+        else
+        {
+          cccl_increment_t incr{.unsigned_offset = offset};
+          (*host_advance_fn_p)(ptr, incr);
+        }
+      }
+      else
+      {
+        throw std::runtime_error("Attempt to increment iterator from host, but host advance function is not defined");
+      }
+    }
+  }
+};

cccl_upstream/c/parallel/src/util/nvjitlink.h

@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <cstdio>
+#include <format>
+#include <memory>
+#include <utility>
+
+#include <nvrtc/nvjitlink_helper.h>
+#include <util/errors.h>
+
+// Links LTO-IR blobs via nvJitLink → returns SASS cubin (PTX fallback if SASS unavailable).
+// Caller owns the returned buffer.
+[[nodiscard]] inline std::pair<std::unique_ptr<char[]>, size_t>
+nvjitlink_link(const void** blobs, const size_t* sizes, size_t num, int cc_major, int cc_minor)
+{
+  const std::string arch = std::format("-arch=sm_{}{}", cc_major, cc_minor);
+  const char* lopts[]    = {"-lto", arch.c_str()};
+
+  nvJitLinkHandle h{};
+  check(nvJitLinkCreate(&h, 2, lopts));
+
+  auto cleanup = [&]() {
+    if (h)
+    {
+      nvJitLinkDestroy(&h);
+      h = nullptr;
+    }
+  };
+
+  try
+  {
+    for (size_t i = 0; i < num; ++i)
+    {
+      if (blobs[i] && sizes[i] > 0)
+      {
+        check(nvJitLinkAddData(h, NVJITLINK_INPUT_ANY, blobs[i], sizes[i], "serialization_input"));
+      }
+    }
+
+    auto rc = nvJitLinkComplete(h);
+
+    size_t log_size = 0;
+    check(nvJitLinkGetErrorLogSize(h, &log_size));
+    if (log_size > 1)
+    {
+      auto log = std::make_unique<char[]>(log_size);
+      check(nvJitLinkGetErrorLog(h, log.get()));
+      fprintf(stderr, "%s\n", log.get());
+    }
+    check(rc);
+
+    size_t cubin_size = 0;
+    bool use_ptx      = (nvJitLinkGetLinkedCubinSize(h, &cubin_size) != NVJITLINK_SUCCESS);
+    if (use_ptx)
+    {
+      check(nvJitLinkGetLinkedPtxSize(h, &cubin_size));
+    }
+    auto cubin = std::make_unique<char[]>(cubin_size);
+    if (use_ptx)
+    {
+      check(nvJitLinkGetLinkedPtx(h, cubin.get()));
+    }
+    else
+    {
+      check(nvJitLinkGetLinkedCubin(h, cubin.get()));
+    }
+
+    cleanup();
+    return {std::move(cubin), cubin_size};
+  }
+  catch (...)
+  {
+    cleanup();
+    throw;
+  }
+}

cccl_upstream/c/parallel/src/util/scan_tile_state.cu

@@ -0,0 +1,89 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <cuda/__cccl_config>
+
+// g++-15 errors in the regex implementation:
+// /usr/include/c++/15/bits/regex_executor.h:243:80: error: argument 1 value is zero [-Werror=alloc-zero]
+//   243 |           _State_info(_StateIdT __start, size_t __n)
+//       |                                                                                ^
+_CCCL_DIAG_SUPPRESS_GCC("-Walloc-zero")
+
+#include <format>
+#include <optional>
+#include <regex>
+
+#include "scan_tile_state.h"
+
+// TODO: NVRTC doesn't currently support extracting basic type
+// information (e.g., type sizes and alignments) from compiled
+// LTO-IR. So we separately compile a small PTX file that defines the
+// necessary types and constants and grep it for the required
+// information. If/when NVRTC adds these features, we can remove this
+// extra compilation step and get the information directly from the
+// LTO-IR.
+static constexpr auto ptx_u64_assignment_regex = R"(\.visible\s+\.global\s+\.align\s+\d+\s+\.u64\s+{}\s*=\s*(\d+);)";
+
+std::optional<size_t> find_size_t(char* ptx, std::string_view name)
+{
+  std::regex regex(std::format(ptx_u64_assignment_regex, name));
+  std::cmatch match;
+  if (std::regex_search(ptx, match, regex))
+  {
+    auto result = std::stoi(match[1].str());
+    return result;
+  }
+  return std::nullopt;
+}
+
+std::pair<size_t, size_t> get_tile_state_bytes_per_tile(
+  cccl_type_info accum_t,
+  const std::string& accum_cpp,
+  const char** ptx_args,
+  size_t num_ptx_args,
+  const std::string& arch)
+{
+  constexpr size_t num_ptx_lto_args       = 3;
+  const char* ptx_lopts[num_ptx_lto_args] = {"-lto", arch.c_str(), "-ptx"};
+
+  constexpr std::string_view ptx_src_template = R"XXX(
+        #include <cub/agent/single_pass_scan_operators.cuh>
+        #include <cub/util_type.cuh>
+        struct __align__({1}) storage_t {{
+           char data[{0}];
+        }};
+        __device__ size_t description_bytes_per_tile = cub::ScanTileState<{2}>::description_bytes_per_tile;
+        __device__ size_t payload_bytes_per_tile = cub::ScanTileState<{2}>::payload_bytes_per_tile;
+        )XXX";
+
+  const std::string ptx_src = std::format(ptx_src_template, accum_t.size, accum_t.alignment, accum_cpp);
+  auto compile_result =
+    begin_linking_nvrtc_program(num_ptx_lto_args, ptx_lopts)
+      ->add_program(nvrtc_translation_unit{ptx_src.c_str(), "tile_state_info"})
+      ->compile_program({ptx_args, num_ptx_args})
+      ->link_program()
+      ->finalize_program();
+  auto ptx_code = compile_result.data.get();
+
+  size_t description_bytes_per_tile;
+  size_t payload_bytes_per_tile;
+  auto maybe_description_bytes_per_tile = find_size_t(ptx_code, "description_bytes_per_tile");
+  if (maybe_description_bytes_per_tile)
+  {
+    description_bytes_per_tile = maybe_description_bytes_per_tile.value();
+  }
+  else
+  {
+    throw std::runtime_error("Failed to find description_bytes_per_tile in PTX");
+  }
+  payload_bytes_per_tile = find_size_t(ptx_code, "payload_bytes_per_tile").value_or(0);
+
+  return {description_bytes_per_tile, payload_bytes_per_tile};
+}

cccl_upstream/c/parallel/src/util/scan_tile_state.h

@@ -0,0 +1,69 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <cub/agent/single_pass_scan_operators.cuh>
+
+#include "cccl/c/types.h"
+#include <nvrtc/command_list.h>
+
+struct scan_tile_state
+{
+  // scan_tile_state implements the same (host) interface as cub::ScanTileStateT, except
+  // that it accepts the acummulator type as a runtime parameter rather than being
+  // templated on it.
+  //
+  // Both specializations ScanTileStateT<T, true> and ScanTileStateT<T, false> - where the
+  // bool parameter indicates whether `T` is primitive - are combined into a single type.
+
+  void* d_tile_status; // d_tile_descriptors
+  void* d_tile_partial;
+  void* d_tile_inclusive;
+
+  size_t description_bytes_per_tile;
+  size_t payload_bytes_per_tile;
+
+  scan_tile_state(size_t description_bytes_per_tile, size_t payload_bytes_per_tile)
+      : d_tile_status(nullptr)
+      , d_tile_partial(nullptr)
+      , d_tile_inclusive(nullptr)
+      , description_bytes_per_tile(description_bytes_per_tile)
+      , payload_bytes_per_tile(payload_bytes_per_tile)
+  {}
+
+  cudaError_t Init(int num_tiles, void* d_temp_storage, size_t temp_storage_bytes)
+  {
+    void* allocations[3] = {};
+    auto status          = cub::detail::tile_state_init(
+      description_bytes_per_tile, payload_bytes_per_tile, num_tiles, d_temp_storage, temp_storage_bytes, allocations);
+    if (status != cudaSuccess)
+    {
+      return status;
+    }
+    d_tile_status    = allocations[0];
+    d_tile_partial   = allocations[1];
+    d_tile_inclusive = allocations[2];
+    return cudaSuccess;
+  }
+
+  cudaError_t AllocationSize(int num_tiles, size_t& temp_storage_bytes) const
+  {
+    return cub::detail::tile_state_allocation_size(
+      temp_storage_bytes, description_bytes_per_tile, payload_bytes_per_tile, num_tiles);
+  }
+};
+
+std::pair<size_t, size_t> get_tile_state_bytes_per_tile(
+  cccl_type_info accum_t,
+  const std::string& accum_cpp,
+  const char** ptx_args,
+  size_t num_ptx_args,
+  const std::string& arch);

cccl_upstream/c/parallel/src/util/serialization.h

@@ -0,0 +1,263 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA Core Compute Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <format>
+#include <memory>
+#include <stdexcept>
+#include <string_view>
+#include <type_traits>
+#include <vector>
+
+#include <cccl/c/serialization.h>
+#include <cccl/c/types.h>
+
+namespace cccl::serialization
+{
+// Opaque 8-byte marker identifying a CCCL serialization blob.
+inline constexpr char k_blob_magic[8] = {'C', 'C', 'C', 'L', 'S', 'E', 'R', '1'};
+
+// Fixed-layout header at the start of every blob. Packed POD; layout is
+// part of the on-disk format, do not reorder. No version field is carried
+// here: blob compatibility is not handled at this layer and must be managed
+// by the caller.
+struct blob_header
+{
+  char magic[8]; // k_blob_magic
+  uint32_t algo_tag; // cccl_serialization_algo_t
+  uint32_t payload_kind; // cccl_payload_kind_t
+  uint32_t cc; // cc_major*10 + cc_minor
+};
+static_assert(sizeof(blob_header) == 20, "blob_header layout must be stable");
+
+// Append-only byte buffer used by *_serialize implementations.
+// Owns a std::vector<char> internally; release() hands back a heap buffer
+// allocated with new[] (matching cccl_serialization_buffer_free, which does delete[]).
+class buffer_writer
+{
+  std::vector<char> data;
+
+public:
+  void write_bytes(const void* p, size_t n)
+  {
+    if (n == 0)
+    {
+      return;
+    }
+    const char* src = static_cast<const char*>(p);
+    data.insert(data.end(), src, src + n);
+  }
+
+  template <class T>
+  void write_pod(const T& v)
+  {
+    static_assert(std::is_trivially_copyable_v<T>, "write_pod requires trivially-copyable type");
+    write_bytes(&v, sizeof(T));
+  }
+
+  // Writes a length-prefixed string. Length is uint64_t. nullptr writes as length=0.
+  void write_cstring(const char* s)
+  {
+    const uint64_t n = (s == nullptr) ? 0 : std::strlen(s);
+    write_pod<uint64_t>(n);
+    if (n > 0)
+    {
+      write_bytes(s, n);
+    }
+  }
+
+  // Writes a length-prefixed byte blob.
+  void write_blob(const void* p, size_t n)
+  {
+    write_pod<uint64_t>(n);
+    if (n > 0)
+    {
+      write_bytes(p, n);
+    }
+  }
+
+  size_t size() const noexcept
+  {
+    return data.size();
+  }
+
+  // Hands back ownership as a new[]'d buffer. After release() the writer is empty.
+  void release(void** out_buf, size_t* out_size)
+  {
+    const size_t n = data.size();
+    auto p         = std::make_unique<char[]>(n);
+    if (n > 0)
+    {
+      std::memcpy(p.get(), data.data(), n);
+    }
+    data.clear();
+    *out_buf  = p.release();
+    *out_size = n;
+  }
+};
+
+// Bounds-checked byte buffer reader used by *_deserialize implementations.
+// Borrows the input buffer; allocations it produces (via read_cstring_dup,
+// read_blob_new) are owned by the caller.
+class buffer_reader
+{
+  const char* pos;
+  size_t nrem;
+
+public:
+  buffer_reader(const void* buf, size_t size)
+      : pos(static_cast<const char*>(buf))
+      , nrem(size)
+  {}
+
+  void read_bytes(void* out, size_t n)
+  {
+    if (n > nrem)
+    {
+      throw std::runtime_error("serialization blob truncated");
+    }
+    std::memcpy(out, pos, n);
+    pos += n;
+    nrem -= n;
+  }
+
+  template <class T>
+  T read_pod()
+  {
+    static_assert(std::is_trivially_copyable_v<T>, "read_pod requires trivially-copyable type");
+    T v;
+    read_bytes(&v, sizeof(T));
+    return v;
+  }
+
+  // Reads a length-prefixed string and returns a fresh new[]'d copy
+  // (always nul-terminated). Length=0 returns nullptr.
+  char* read_cstring_dup()
+  {
+    const uint64_t n = read_pod<uint64_t>();
+    if (n == 0)
+    {
+      return nullptr;
+    }
+    if (n > nrem)
+    {
+      throw std::runtime_error("serialization blob truncated (cstring)");
+    }
+    auto out = std::make_unique<char[]>(n + 1);
+    std::memcpy(out.get(), pos, n);
+    out[n] = '\0';
+    pos += n;
+    nrem -= n;
+    return out.release();
+  }
+
+  // Reads a length-prefixed byte blob into a fresh new[]'d buffer.
+  // Length=0 returns nullptr with *out_size=0.
+  void read_blob_new(void** out_buf, size_t* out_size)
+  {
+    const uint64_t n = read_pod<uint64_t>();
+    if (n > nrem)
+    {
+      throw std::runtime_error("serialization blob truncated (blob)");
+    }
+    if (n == 0)
+    {
+      *out_buf  = nullptr;
+      *out_size = 0;
+      return;
+    }
+    auto out = std::make_unique<char[]>(n);
+    std::memcpy(out.get(), pos, n);
+    pos += n;
+    nrem -= n;
+    *out_buf  = out.release();
+    *out_size = n;
+  }
+
+  // Reads a length-prefixed POD blob directly into an existing pointer
+  // (allocated by the caller as new T). Used for runtime_policy where the
+  // target type's allocator must match the algorithm's cleanup path.
+  void read_into(void* dest, size_t expected_size)
+  {
+    const uint64_t n = read_pod<uint64_t>();
+    if (n != expected_size)
+    {
+      throw std::runtime_error("serialization blob runtime_policy size mismatch");
+    }
+    if (n > 0)
+    {
+      read_bytes(dest, n);
+    }
+  }
+
+  size_t remaining() const noexcept
+  {
+    return nrem;
+  }
+};
+
+// Writes the standard blob header.
+inline void write_header(buffer_writer& w, cccl_serialization_algo_t algo_tag, cccl_payload_kind_t kind, int cc)
+{
+  blob_header h{};
+  std::memcpy(h.magic, k_blob_magic, sizeof(k_blob_magic));
+  h.algo_tag     = static_cast<uint32_t>(algo_tag);
+  h.payload_kind = static_cast<uint32_t>(kind);
+  h.cc           = static_cast<uint32_t>(cc);
+  w.write_pod(h);
+}
+
+// Reads + validates a blob header. Throws on magic / algo_tag mismatch.
+// Returns the parsed header for the caller to use (payload_kind, cc).
+inline blob_header read_and_validate_header(buffer_reader& r, cccl_serialization_algo_t expected_algo)
+{
+  const auto h = r.read_pod<blob_header>();
+  if (std::memcmp(h.magic, k_blob_magic, sizeof(k_blob_magic)) != 0)
+  {
+    throw std::runtime_error("serialization blob: bad magic");
+  }
+  if (h.algo_tag != static_cast<uint32_t>(expected_algo))
+  {
+    throw std::runtime_error("serialization blob: wrong algorithm");
+  }
+  if (h.payload_kind != CCCL_PAYLOAD_LTOIR && h.payload_kind != CCCL_PAYLOAD_CUBIN)
+  {
+    throw std::runtime_error("serialization blob: unknown payload kind");
+  }
+  return h;
+}
+
+// Serializes a cccl_type_info as a fixed POD record.
+inline void write_type_info(buffer_writer& w, const cccl_type_info& t)
+{
+  w.write_pod<uint64_t>(static_cast<uint64_t>(t.size));
+  w.write_pod<uint64_t>(static_cast<uint64_t>(t.alignment));
+  w.write_pod<uint32_t>(static_cast<uint32_t>(t.type));
+}
+
+inline cccl_type_info read_type_info(buffer_reader& r)
+{
+  cccl_type_info t{};
+  t.size            = static_cast<size_t>(r.read_pod<uint64_t>());
+  t.alignment       = static_cast<size_t>(r.read_pod<uint64_t>());
+  const auto type_v = r.read_pod<uint32_t>();
+  if (type_v > static_cast<uint32_t>(CCCL_BOOLEAN))
+  {
+    throw std::runtime_error(std::format("serialization blob: invalid type enum ({})", type_v));
+  }
+  t.type = static_cast<cccl_type_enum>(type_v);
+  return t;
+}
+} // namespace cccl::serialization

cccl_upstream/c/parallel/src/util/tuning.cpp

@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include "tuning.h"
+
+#include <algorithm>
+
+int nominal_4b_items_to_items(int nominal_4b_items_per_thread, int key_size)
+{
+  return std::clamp(nominal_4b_items_per_thread * 4 / key_size, 1, nominal_4b_items_per_thread);
+}

cccl_upstream/c/parallel/src/util/tuning.h

@@ -0,0 +1,13 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+int nominal_4b_items_to_items(int nominal_4b_items_per_thread, int key_size);

cccl_upstream/c/parallel/src/util/types.h

@@ -0,0 +1,196 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <cub/device/dispatch/tuning/common.cuh>
+
+#include <cuda/std/cstdint>
+
+#include <cstring>
+#include <memory>
+#include <string>
+#include <string_view>
+
+#if defined(_WIN32)
+#  include <mutex>
+#endif // _WIN32
+
+#include "errors.h"
+#include <cccl/c/types.h>
+
+struct storage_t;
+struct input_storage_t;
+struct output_storage_t;
+struct items_storage_t; // Used in merge_sort
+
+// On Windows, nvrtcGetTypeName calls UnDecorateSymbolName from Dbghelp.dll,
+// which, for certain input types, returns string representations that nvcc
+// balks on (e.g. `long long` becomes `__int64`).  This helper function looks
+// for these unsupported types and converts them to nvcc-compatible types.
+// The method signature is kept identical to `nvrtcGetTypeName` so that this
+// helper can be used as a drop-in replacement.
+#if defined(_WIN32)
+// The Windows nvrtcGetTypeName path demangles through DbgHelp's
+// UnDecorateSymbolName, which Microsoft documents as single-threaded: concurrent
+// calls "will likely result in unexpected behavior or memory corruption." Under
+// free-threaded Python, distinct-key build storms resolve type names on multiple
+// threads at once, and the returned names get swapped/corrupted, producing bogus
+// NVRTC name expressions (e.g. a transform kernel instantiated with a reduce
+// policy). Serialize every call through one process-wide mutex; the `inline`
+// variable is a single instance shared across all `cccl_type_name_from_nvrtc<T>`
+// instantiations and translation units.
+inline std::mutex nvrtc_type_name_mutex;
+#endif // _WIN32
+
+template <typename T>
+nvrtcResult cccl_type_name_from_nvrtc(std::string* result)
+{
+#if defined(_WIN32)
+  const std::lock_guard<std::mutex> lock(nvrtc_type_name_mutex);
+#endif // _WIN32
+  if (const nvrtcResult res = nvrtcGetTypeName<T>(result); res != NVRTC_SUCCESS)
+  {
+    return res;
+  }
+
+  if (result->find("unsigned __int64") != std::string::npos)
+  {
+    *result = "::cuda::std::uint64_t";
+  }
+  else if (result->find("__int64") != std::string::npos)
+  {
+    *result = "::cuda::std::int64_t";
+  }
+
+  return NVRTC_SUCCESS;
+}
+
+template <typename StorageT = storage_t>
+std::string cccl_type_enum_to_name(cccl_type_enum type, bool is_pointer = false)
+{
+  std::string result;
+
+  switch (type)
+  {
+    case cccl_type_enum::CCCL_INT8:
+      result = "::cuda::std::int8_t";
+      break;
+    case cccl_type_enum::CCCL_INT16:
+      result = "::cuda::std::int16_t";
+      break;
+    case cccl_type_enum::CCCL_INT32:
+      result = "::cuda::std::int32_t";
+      break;
+    case cccl_type_enum::CCCL_INT64:
+      result = "::cuda::std::int64_t";
+      break;
+    case cccl_type_enum::CCCL_UINT8:
+      result = "::cuda::std::uint8_t";
+      break;
+    case cccl_type_enum::CCCL_UINT16:
+      result = "::cuda::std::uint16_t";
+      break;
+    case cccl_type_enum::CCCL_UINT32:
+      result = "::cuda::std::uint32_t";
+      break;
+    case cccl_type_enum::CCCL_UINT64:
+      result = "::cuda::std::uint64_t";
+      break;
+    case cccl_type_enum::CCCL_FLOAT16:
+#if _CCCL_HAS_NVFP16()
+      result = "__half";
+      break;
+#else
+      throw std::runtime_error("float16 is not supported");
+#endif
+    case cccl_type_enum::CCCL_FLOAT32:
+      result = "float";
+      break;
+    case cccl_type_enum::CCCL_FLOAT64:
+      result = "double";
+      break;
+    case cccl_type_enum::CCCL_STORAGE:
+      check(cccl_type_name_from_nvrtc<StorageT>(&result));
+      break;
+    case cccl_type_enum::CCCL_BOOLEAN:
+      result = "bool";
+      break;
+  }
+
+  if (is_pointer)
+  {
+    result += "*";
+  }
+
+  return result;
+}
+
+inline constexpr cub::detail::type_t cccl_type_enum_to_cub_type(cccl_type_enum type)
+{
+  switch (type)
+  {
+    case CCCL_BOOLEAN:
+      return cub::detail::type_t::boolean;
+    case CCCL_INT8:
+      return cub::detail::type_t::int8;
+    case CCCL_INT16:
+      return cub::detail::type_t::int16;
+    case CCCL_INT32:
+      return cub::detail::type_t::int32;
+    case CCCL_INT64:
+      return cub::detail::type_t::int64;
+    case CCCL_UINT8:
+      return cub::detail::type_t::uint8;
+    case CCCL_UINT16:
+      return cub::detail::type_t::uint16;
+    case CCCL_UINT32:
+      return cub::detail::type_t::uint32;
+    case CCCL_UINT64:
+      return cub::detail::type_t::uint64;
+    case CCCL_FLOAT32:
+      return cub::detail::type_t::float32;
+    case CCCL_FLOAT64:
+      return cub::detail::type_t::float64;
+    case CCCL_FLOAT16:
+    case CCCL_STORAGE:
+    default:
+      return cub::detail::type_t::other;
+  }
+}
+
+inline char* duplicate_c_string(std::string_view s)
+{
+  auto p = std::make_unique<char[]>(s.size() + 1);
+  std::memcpy(p.get(), s.data(), s.size());
+  p[s.size()] = '\0';
+  return p.release();
+}
+
+// A custom op has a name but no LTOIR — kernel-only compile mode leaves the op unresolved.
+inline bool is_custom_op(cccl_op_t op)
+{
+  return op.code_size == 0 && op.name != nullptr && op.name[0] != '\0';
+}
+
+inline constexpr cub::detail::op_kind_t cccl_op_kind_to_cub_op(cccl_op_kind_t type)
+{
+  switch (type)
+  {
+    case CCCL_PLUS:
+      return cub::detail::op_kind_t::plus;
+    case CCCL_MINIMUM:
+      return cub::detail::op_kind_t::min;
+    case CCCL_MAXIMUM:
+      return cub::detail::op_kind_t::max;
+    default:
+      return cub::detail::op_kind_t::other;
+  }
+}