[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/thrust/benchmarks/bench/set_operations/base.cuh

@@ -0,0 +1,54 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#pragma once
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/set_operations.h>
+#include <thrust/sort.h>
+
+#include "nvbench_helper.cuh"
+
+template <typename T, typename OpT>
+static void basic(nvbench::state& state, nvbench::type_list<T>, OpT op)
+{
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto size_ratio     = static_cast<std::size_t>(state.get_int64("SizeRatio"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  const auto elements_in_A = static_cast<std::size_t>(static_cast<double>(size_ratio * elements) / 100.0f);
+
+  thrust::device_vector<T> input = generate(elements, entropy);
+  thrust::device_vector<T> output(elements);
+
+  thrust::sort(input.begin(), input.begin() + elements_in_A);
+  thrust::sort(input.begin() + elements_in_A, input.end());
+
+  caching_allocator_t alloc;
+  // not a warm-up run, we need to run once to determine the size of the output
+  const auto result_ends =
+    op(policy(alloc),
+       input.cbegin(),
+       input.cbegin() + elements_in_A,
+       input.cbegin() + elements_in_A,
+       input.cend(),
+       output.begin());
+  const std::size_t elements_in_AB = ::cuda::std::distance(output.begin(), result_ends);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(elements_in_AB);
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               op(policy(alloc, launch),
+                  input.cbegin(),
+                  input.cbegin() + elements_in_A,
+                  input.cbegin() + elements_in_A,
+                  input.cend(),
+                  output.begin());
+             });
+}
+
+using types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t>;

cccl_upstream/thrust/benchmarks/bench/set_operations/by_key.cuh

@@ -0,0 +1,66 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#pragma once
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/set_operations.h>
+#include <thrust/sort.h>
+
+#include "nvbench_helper.cuh"
+
+template <class KeyT, class ValueT, class OpT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT>, OpT op)
+{
+  const auto elements       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto size_ratio     = static_cast<std::size_t>(state.get_int64("SizeRatio"));
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  const auto elements_in_A = static_cast<std::size_t>(static_cast<double>(size_ratio * elements) / 100.0f);
+
+  thrust::device_vector<KeyT> in_keys = generate(elements, entropy);
+  thrust::device_vector<KeyT> out_keys(elements);
+
+  thrust::device_vector<ValueT> in_vals(elements);
+  thrust::device_vector<ValueT> out_vals(elements);
+
+  thrust::sort(in_keys.begin(), in_keys.begin() + elements_in_A);
+  thrust::sort(in_keys.begin() + elements_in_A, in_keys.end());
+
+  caching_allocator_t alloc;
+  // not a warm-up run, we need to run once to determine the size of the output
+  auto result_ends = op(
+    policy(alloc),
+    in_keys.cbegin(),
+    in_keys.cbegin() + elements_in_A,
+    in_keys.cbegin() + elements_in_A,
+    in_keys.cend(),
+    in_vals.cbegin(),
+    in_vals.cbegin() + elements_in_A,
+    out_keys.begin(),
+    out_vals.begin());
+  const std::size_t elements_in_AB = ::cuda::std::distance(out_keys.begin(), result_ends.first);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_writes<KeyT>(elements_in_AB);
+  state.add_global_memory_reads<ValueT>(OpT::read_all_values ? elements : elements_in_A);
+  state.add_global_memory_writes<ValueT>(elements_in_AB);
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               op(policy(alloc, launch),
+                  in_keys.cbegin(),
+                  in_keys.cbegin() + elements_in_A,
+                  in_keys.cbegin() + elements_in_A,
+                  in_keys.cend(),
+                  in_vals.cbegin(),
+                  in_vals.cbegin() + elements_in_A,
+                  out_keys.begin(),
+                  out_vals.begin());
+             });
+}
+
+using key_types   = nvbench::type_list<int8_t, int16_t, int32_t, int64_t>;
+using value_types = nvbench::type_list<int8_t, int64_t>;

cccl_upstream/thrust/benchmarks/bench/set_operations/difference.cu

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "base.cuh"
+
+struct op_t
+{
+  template <class PolicyT, class InputIterator1, class InputIterator2, class OutputIterator>
+  __host__ OutputIterator operator()(
+    const PolicyT& policy,
+    InputIterator1 first1,
+    InputIterator1 last1,
+    InputIterator2 first2,
+    InputIterator2 last2,
+    OutputIterator result) const
+  {
+    return thrust::set_difference(policy, first1, last1, first2, last2, result);
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/difference_by_key.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "by_key.cuh"
+
+struct op_t
+{
+  static constexpr bool read_all_values = true;
+
+  template <class PolicyT,
+            class InputIterator1,
+            class InputIterator2,
+            class InputIterator3,
+            class InputIterator4,
+            class OutputIterator1,
+            class OutputIterator2>
+  __host__ cuda::std::pair<OutputIterator1, OutputIterator2> operator()(
+    const PolicyT& policy,
+    InputIterator1 keys_first1,
+    InputIterator1 keys_last1,
+    InputIterator2 keys_first2,
+    InputIterator2 keys_last2,
+    InputIterator3 values_first1,
+    InputIterator4 values_first2,
+    OutputIterator1 keys_result,
+    OutputIterator2 values_result) const
+  {
+    return thrust::set_difference_by_key(
+      policy, keys_first1, keys_last1, keys_first2, keys_last2, values_first1, values_first2, keys_result, values_result);
+  }
+};
+
+template <class KeyT, class ValueT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/intersection.cu

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "base.cuh"
+
+struct op_t
+{
+  template <class PolicyT, class InputIterator1, class InputIterator2, class OutputIterator>
+  __host__ OutputIterator operator()(
+    const PolicyT& policy,
+    InputIterator1 first1,
+    InputIterator1 last1,
+    InputIterator2 first2,
+    InputIterator2 last2,
+    OutputIterator result) const
+  {
+    return thrust::set_intersection(policy, first1, last1, first2, last2, result);
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/intersection_by_key.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "by_key.cuh"
+
+struct op_t
+{
+  static constexpr bool read_all_values = false;
+
+  template <class PolicyT,
+            class InputIterator1,
+            class InputIterator2,
+            class InputIterator3,
+            class InputIterator4,
+            class OutputIterator1,
+            class OutputIterator2>
+  __host__ cuda::std::pair<OutputIterator1, OutputIterator2> operator()(
+    const PolicyT& policy,
+    InputIterator1 keys_first1,
+    InputIterator1 keys_last1,
+    InputIterator2 keys_first2,
+    InputIterator2 keys_last2,
+    InputIterator3 values_first1,
+    InputIterator4 /* values_first2 */,
+    OutputIterator1 keys_result,
+    OutputIterator2 values_result) const
+  {
+    return thrust::set_intersection_by_key(
+      policy, keys_first1, keys_last1, keys_first2, keys_last2, values_first1, keys_result, values_result);
+  }
+};
+
+template <class KeyT, class ValueT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/symmetric_difference.cu

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "base.cuh"
+
+struct op_t
+{
+  template <class PolicyT, class InputIterator1, class InputIterator2, class OutputIterator>
+  __host__ OutputIterator operator()(
+    const PolicyT& policy,
+    InputIterator1 first1,
+    InputIterator1 last1,
+    InputIterator2 first2,
+    InputIterator2 last2,
+    OutputIterator result) const
+  {
+    return thrust::set_symmetric_difference(policy, first1, last1, first2, last2, result);
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/symmetric_difference_by_key.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "by_key.cuh"
+
+struct op_t
+{
+  static constexpr bool read_all_values = true;
+
+  template <class PolicyT,
+            class InputIterator1,
+            class InputIterator2,
+            class InputIterator3,
+            class InputIterator4,
+            class OutputIterator1,
+            class OutputIterator2>
+  __host__ cuda::std::pair<OutputIterator1, OutputIterator2> operator()(
+    const PolicyT& policy,
+    InputIterator1 keys_first1,
+    InputIterator1 keys_last1,
+    InputIterator2 keys_first2,
+    InputIterator2 keys_last2,
+    InputIterator3 values_first1,
+    InputIterator4 values_first2,
+    OutputIterator1 keys_result,
+    OutputIterator2 values_result) const
+  {
+    return thrust::set_symmetric_difference_by_key(
+      policy, keys_first1, keys_last1, keys_first2, keys_last2, values_first1, values_first2, keys_result, values_result);
+  }
+};
+
+template <class KeyT, class ValueT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/union.cu

@@ -0,0 +1,32 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "base.cuh"
+
+struct op_t
+{
+  template <class PolicyT, class InputIterator1, class InputIterator2, class OutputIterator>
+  __host__ OutputIterator operator()(
+    const PolicyT& policy,
+    InputIterator1 first1,
+    InputIterator1 last1,
+    InputIterator2 first2,
+    InputIterator2 last2,
+    OutputIterator result) const
+  {
+    return thrust::set_union(policy, first1, last1, first2, last2, result);
+  }
+};
+
+template <typename T>
+static void basic(nvbench::state& state, nvbench::type_list<T> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(types))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});

cccl_upstream/thrust/benchmarks/bench/set_operations/union_by_key.cu

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include "by_key.cuh"
+
+struct op_t
+{
+  static constexpr bool read_all_values = true;
+
+  template <class PolicyT,
+            class InputIterator1,
+            class InputIterator2,
+            class InputIterator3,
+            class InputIterator4,
+            class OutputIterator1,
+            class OutputIterator2>
+  __host__ cuda::std::pair<OutputIterator1, OutputIterator2> operator()(
+    const PolicyT& policy,
+    InputIterator1 keys_first1,
+    InputIterator1 keys_last1,
+    InputIterator2 keys_first2,
+    InputIterator2 keys_last2,
+    InputIterator3 values_first1,
+    InputIterator4 values_first2,
+    OutputIterator1 keys_result,
+    OutputIterator2 values_result) const
+  {
+    return thrust::set_union_by_key(
+      policy, keys_first1, keys_last1, keys_first2, keys_last2, values_first1, values_first2, keys_result, values_result);
+  }
+};
+
+template <class KeyT, class ValueT>
+static void basic(nvbench::state& state, nvbench::type_list<KeyT, ValueT> tl)
+{
+  basic(state, tl, op_t{});
+}
+
+NVBENCH_BENCH_TYPES(basic, NVBENCH_TYPE_AXES(key_types, value_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.201"})
+  .add_int64_axis("SizeRatio", {25, 50, 75});