[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/cub/benchmarks/bench/select/flagged.cu

@@ -0,0 +1,116 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_select.cuh>
+
+#include <thrust/count.h>
+
+#include <cuda/std/algorithm>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+template <typename InputT>
+struct bench_policy_selector
+{
+  [[nodiscard]] _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::SelectPolicy
+  {
+    return {cub::SelectAlgorithm::lookback,
+            {TUNE_THREADS_PER_BLOCK,
+             TUNE_ITEMS_PER_THREAD,
+             (TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE),
+             (TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : cub::LOAD_CA),
+             cub::BLOCK_SCAN_WARP_SCANS,
+             lookback_delay_policy}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T, typename InPlace>
+void select(nvbench::state& state, nvbench::type_list<T, InPlace>)
+{
+  using offset_t = int64_t;
+
+  // Retrieve axis parameters
+  const auto elements       = state.get_int64("Elements{io}");
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  auto generator = generate(elements, entropy);
+
+  thrust::device_vector<T> in       = generator;
+  thrust::device_vector<bool> flags = generator;
+  thrust::device_vector<offset_t> num_selected(1);
+
+  // TODO Extract into helper TU
+  const auto selected_elements = thrust::count(flags.cbegin(), flags.cend(), true);
+  thrust::device_vector<T> out(selected_elements, thrust::no_init);
+
+  T* d_in                  = thrust::raw_pointer_cast(in.data());
+  T* d_out                 = thrust::raw_pointer_cast(out.data());
+  const bool* d_flags      = thrust::raw_pointer_cast(flags.data());
+  offset_t* d_num_selected = thrust::raw_pointer_cast(num_selected.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_reads<bool>(elements);
+  state.add_global_memory_writes<T>(selected_elements);
+  state.add_global_memory_writes<offset_t>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<T>{})
+#endif // !TUNE_BASE
+    );
+    if constexpr (InPlace::value)
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSelect::Flagged,
+        "DeviceSelect::Flagged failed",
+        d_in,
+        d_flags,
+        d_num_selected,
+        static_cast<offset_t>(elements),
+        env);
+    }
+    else
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSelect::Flagged,
+        "DeviceSelect::Flagged failed",
+        static_cast<const T*>(d_in),
+        d_flags,
+        d_out,
+        d_num_selected,
+        static_cast<offset_t>(elements),
+        env);
+    }
+  });
+}
+
+using ::cuda::std::false_type;
+using ::cuda::std::true_type;
+#ifdef TUNE_InPlace
+using is_in_place = nvbench::type_list<TUNE_InPlace>; // expands to "false_type" or "true_type"
+#else // !defined(TUNE_InPlace)
+using is_in_place = nvbench::type_list<false_type, true_type>;
+#endif // TUNE_InPlace
+
+NVBENCH_BENCH_TYPES(select, NVBENCH_TYPE_AXES(fundamental_types, is_in_place))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "InPlace{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.000"});

cccl_upstream/cub/benchmarks/bench/select/if.cu

@@ -0,0 +1,117 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_select.cuh>
+
+#include <thrust/count.h>
+
+#include <cuda/std/algorithm>
+
+#include <limits>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+template <typename InputT>
+struct bench_policy_selector
+{
+  [[nodiscard]] _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::SelectPolicy
+  {
+    return {cub::SelectAlgorithm::lookback,
+            {TUNE_THREADS_PER_BLOCK,
+             TUNE_ITEMS_PER_THREAD,
+             (TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE),
+             (TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : cub::LOAD_CA),
+             cub::BLOCK_SCAN_WARP_SCANS,
+             lookback_delay_policy}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T, typename InPlace>
+void select(nvbench::state& state, nvbench::type_list<T, InPlace>)
+{
+  using offset_t    = int64_t;
+  using select_op_t = less_then_t<T>;
+
+  // Retrieve axis parameters
+  const auto elements       = state.get_int64("Elements{io}");
+  const bit_entropy entropy = str_to_entropy(state.get_string("Entropy"));
+
+  const T val = lerp_min_max<T>(entropy_to_probability(entropy));
+  select_op_t select_op{val};
+
+  thrust::device_vector<T> in = generate(elements);
+  thrust::device_vector<offset_t> num_selected(1);
+
+  // TODO Extract into helper TU
+  const auto selected_elements = thrust::count_if(in.cbegin(), in.cend(), select_op);
+  thrust::device_vector<T> out(selected_elements, thrust::no_init);
+
+  T* d_in                  = thrust::raw_pointer_cast(in.data());
+  T* d_out                 = thrust::raw_pointer_cast(out.data());
+  offset_t* d_num_selected = thrust::raw_pointer_cast(num_selected.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(selected_elements);
+  state.add_global_memory_writes<offset_t>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<T>{})
+#endif // !TUNE_BASE
+    );
+    if constexpr (InPlace::value)
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSelect::If,
+        "select_if failed",
+        d_in,
+        d_num_selected,
+        static_cast<offset_t>(elements),
+        select_op,
+        env);
+    }
+    else
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSelect::If,
+        "select_if failed",
+        static_cast<const T*>(d_in),
+        d_out,
+        d_num_selected,
+        static_cast<offset_t>(elements),
+        select_op,
+        env);
+    }
+  });
+}
+
+using ::cuda::std::false_type;
+using ::cuda::std::true_type;
+#ifdef TUNE_InPlace
+using is_in_place = nvbench::type_list<TUNE_InPlace>; // expands to "false_type" or "true_type"
+#else // !defined(TUNE_InPlace)
+using is_in_place = nvbench::type_list<false_type, true_type>;
+#endif // TUNE_InPlace
+
+NVBENCH_BENCH_TYPES(select, NVBENCH_TYPE_AXES(fundamental_types, is_in_place))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "InPlace{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.000"});

cccl_upstream/cub/benchmarks/bench/select/unique.cu

@@ -0,0 +1,120 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include <cub/device/device_select.cuh>
+
+#include <cuda/std/algorithm>
+
+#include <limits>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_ITEMS_PER_THREAD ipt 7:24:1
+// %RANGE% TUNE_THREADS_PER_BLOCK tpb 128:1024:32
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+template <typename InputT>
+struct bench_policy_selector
+{
+  [[nodiscard]] _CCCL_API constexpr auto operator()(cuda::compute_capability) const -> cub::SelectPolicy
+  {
+    return {cub::SelectAlgorithm::lookback,
+            {TUNE_THREADS_PER_BLOCK,
+             TUNE_ITEMS_PER_THREAD,
+             (TUNE_TRANSPOSE == 0 ? cub::BLOCK_LOAD_DIRECT : cub::BLOCK_LOAD_WARP_TRANSPOSE),
+             (TUNE_LOAD == 0 ? cub::LOAD_DEFAULT : cub::LOAD_CA),
+             cub::BLOCK_SCAN_WARP_SCANS,
+             lookback_delay_policy}};
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename T, typename InPlace>
+static void unique(nvbench::state& state, nvbench::type_list<T, InPlace>)
+{
+  using offset_t = int64_t;
+
+  // Retrieve axis parameters
+  const auto elements         = state.get_int64("Elements{io}");
+  const auto max_segment_size = state.get_int64("MaxSegSize");
+
+  thrust::device_vector<T> in = generate.uniform.key_segments(elements, /* min_segmented_size */ 1, max_segment_size);
+  thrust::device_vector<T> out(elements, thrust::no_init);
+  thrust::device_vector<offset_t> num_unique_out(1);
+
+  T* d_in                = thrust::raw_pointer_cast(in.data());
+  T* d_out               = thrust::raw_pointer_cast(out.data());
+  offset_t* d_num_unique = thrust::raw_pointer_cast(num_unique_out.data());
+
+  // Get number of unique elements for metrics
+  _CCCL_TRY_CUDA_API(
+    cub::DeviceSelect::Unique,
+    "select_unique failed",
+    d_in,
+    d_out,
+    d_num_unique,
+    static_cast<offset_t>(elements),
+    ::cuda::std::equal_to<>{});
+  cudaDeviceSynchronize();
+  const offset_t num_unique = num_unique_out[0];
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<T>(num_unique);
+  state.add_global_memory_writes<offset_t>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<T>{})
+#endif // !TUNE_BASE
+    );
+    if constexpr (InPlace::value)
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSelect::Unique,
+        "select_unique failed",
+        d_in,
+        d_num_unique,
+        static_cast<offset_t>(elements),
+        ::cuda::std::equal_to<>{},
+        env);
+    }
+    else
+    {
+      _CCCL_TRY_CUDA_API(
+        cub::DeviceSelect::Unique,
+        "select_unique failed",
+        d_in,
+        d_out,
+        d_num_unique,
+        static_cast<offset_t>(elements),
+        ::cuda::std::equal_to<>{},
+        env);
+    }
+  });
+}
+
+using ::cuda::std::false_type;
+using ::cuda::std::true_type;
+#ifdef TUNE_InPlace
+using is_in_place = nvbench::type_list<TUNE_InPlace>; // expands to "false_type" or "true_type"
+#else // !defined(TUNE_InPlace)
+using is_in_place = nvbench::type_list<false_type, true_type>;
+#endif // TUNE_InPlace
+
+NVBENCH_BENCH_TYPES(unique, NVBENCH_TYPE_AXES(fundamental_types, is_in_place))
+  .set_name("base")
+  .set_type_axes_names({"T{ct}", "InPlace{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegSize", {1, 4, 8});

cccl_upstream/cub/benchmarks/bench/select/unique_by_key.cu

@@ -0,0 +1,139 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <cub/device/device_select.cuh>
+
+#include <look_back_helper.cuh>
+#include <nvbench_helper.cuh>
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_TRANSPOSE trp 0:1:1
+// %RANGE% TUNE_LOAD ld 0:1:1
+// %RANGE% TUNE_MAGIC_NS ns 0:2048:4
+// %RANGE% TUNE_DELAY_CONSTRUCTOR_ID dcid 0:7:1
+// %RANGE% TUNE_L2_WRITE_LATENCY_NS l2w 0:1200:5
+
+#if !TUNE_BASE
+#  if TUNE_TRANSPOSE == 0
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_DIRECT
+#  else // TUNE_TRANSPOSE == 1
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_WARP_TRANSPOSE
+#  endif // TUNE_TRANSPOSE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  else // TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+
+struct bench_unique_by_key_policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::UniqueByKeyPolicy
+  {
+    return {TUNE_THREADS,
+            TUNE_ITEMS,
+            TUNE_LOAD_ALGORITHM,
+            TUNE_LOAD_MODIFIER,
+            cub::BLOCK_SCAN_WARP_SCANS,
+            lookback_delay_policy};
+  }
+};
+#endif // !TUNE_BASE
+
+template <class KeyT, class ValueT, class OffsetT>
+static void select(nvbench::state& state, nvbench::type_list<KeyT, ValueT, OffsetT>)
+{
+  using equality_op_t = cuda::std::equal_to<>;
+
+  const auto elements                    = static_cast<std::size_t>(state.get_int64("Elements{io}"));
+  constexpr std::size_t min_segment_size = 1;
+  const std::size_t max_segment_size     = static_cast<std::size_t>(state.get_int64("MaxSegSize"));
+
+  thrust::device_vector<OffsetT> num_runs_out(1);
+  thrust::device_vector<ValueT> in_vals(elements);
+  thrust::device_vector<ValueT> out_vals(elements);
+  thrust::device_vector<KeyT> out_keys(elements);
+  thrust::device_vector<KeyT> in_keys = generate.uniform.key_segments(elements, min_segment_size, max_segment_size);
+
+  const KeyT* d_in_keys   = thrust::raw_pointer_cast(in_keys.data());
+  KeyT* d_out_keys        = thrust::raw_pointer_cast(out_keys.data());
+  const ValueT* d_in_vals = thrust::raw_pointer_cast(in_vals.data());
+  ValueT* d_out_vals      = thrust::raw_pointer_cast(out_vals.data());
+  OffsetT* d_num_runs_out = thrust::raw_pointer_cast(num_runs_out.data());
+
+  const auto num_items = static_cast<OffsetT>(elements);
+
+  // Pre-computation to get num_runs for statistics
+  _CCCL_TRY_CUDA_API(
+    cub::DeviceSelect::UniqueByKey,
+    "UniqueByKey failed",
+    d_in_keys,
+    d_in_vals,
+    d_out_keys,
+    d_out_vals,
+    d_num_runs_out,
+    num_items,
+    equality_op_t{});
+  _CCCL_TRY_CUDA_API(cudaDeviceSynchronize, "Sync failed");
+  const OffsetT num_runs = num_runs_out[0];
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<KeyT>(elements);
+  state.add_global_memory_reads<ValueT>(elements);
+  state.add_global_memory_writes<ValueT>(num_runs);
+  state.add_global_memory_writes<KeyT>(num_runs);
+  state.add_global_memory_writes<OffsetT>(1);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_unique_by_key_policy_selector{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceSelect::UniqueByKey,
+      "UniqueByKey failed",
+      d_in_keys,
+      d_in_vals,
+      d_out_keys,
+      d_out_vals,
+      d_num_runs_out,
+      num_items,
+      equality_op_t{},
+      env);
+  });
+}
+
+using some_offset_types = nvbench::type_list<nvbench::int32_t>;
+
+#ifdef TUNE_KeyT
+using key_types = nvbench::type_list<TUNE_KeyT>;
+#else // !defined(TUNE_KeyT)
+using key_types =
+  nvbench::type_list<int8_t,
+                     int16_t,
+                     int32_t,
+                     int64_t
+#  if _CCCL_HAS_INT128()
+                     ,
+                     int128_t
+#  endif
+                     >;
+#endif // TUNE_KeyT
+
+#ifdef TUNE_ValueT
+using value_types = nvbench::type_list<TUNE_ValueT>;
+#else // !defined(TUNE_ValueT)
+using value_types = all_types;
+#endif // TUNE_ValueT
+
+NVBENCH_BENCH_TYPES(select, NVBENCH_TYPE_AXES(key_types, value_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"KeyT{ct}", "ValueT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_power_of_two_axis("MaxSegSize", {1, 4, 8});