[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/cub/benchmarks/bench/histogram/even.cu

@@ -0,0 +1,85 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <nvbench_helper.cuh>
+
+#include "histogram_common.cuh"
+
+// %RANGE% TUNE_ITEMS ipt 4:28:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_RLE_COMPRESS rle 0:1:1
+// %RANGE% TUNE_WORK_STEALING ws 0:1:1
+// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1
+// %RANGE% TUNE_LOAD ld 0:2:1
+// %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1
+// %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1
+
+template <typename SampleT, typename CounterT, typename OffsetT>
+static void even(nvbench::state& state, nvbench::type_list<SampleT, CounterT, OffsetT>)
+{
+  const auto entropy   = str_to_entropy(state.get_string("Entropy"));
+  const auto elements  = state.get_int64("Elements{io}");
+  const auto num_bins  = state.get_int64("Bins");
+  const int num_levels = static_cast<int>(num_bins) + 1;
+
+  // Skip invalid configurations where LevelT (= SampleT) cannot represent the number of bins
+  if constexpr (cuda::std::is_integral_v<SampleT>)
+  {
+    if (num_bins > static_cast<int64_t>(cuda::std::numeric_limits<SampleT>::max()))
+    {
+      state.skip("Number of bins exceeds what LevelT (= SampleT) can represent");
+      return;
+    }
+  }
+
+  const SampleT lower_level = 0;
+  const SampleT upper_level = get_upper_level<SampleT>(num_bins, elements);
+
+  thrust::device_vector<SampleT> input = generate(elements, entropy, lower_level, upper_level);
+  thrust::device_vector<CounterT> hist(num_bins);
+
+  SampleT* d_input      = thrust::raw_pointer_cast(input.data());
+  CounterT* d_histogram = thrust::raw_pointer_cast(hist.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<SampleT>(elements);
+  state.add_global_memory_writes<CounterT>(num_bins);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<key_t, 1, 1>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceHistogram::HistogramEven,
+      "HistogramEven failed",
+      d_input,
+      d_histogram,
+      num_levels,
+      lower_level,
+      upper_level,
+      static_cast<OffsetT>(elements),
+      env);
+  });
+}
+
+using counter_types     = nvbench::type_list<int32_t>;
+using some_offset_types = nvbench::type_list<int32_t>;
+
+#ifdef TUNE_SampleT
+using sample_types = nvbench::type_list<TUNE_SampleT>;
+#else // !defined(TUNE_SampleT)
+using sample_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double>;
+#endif // TUNE_SampleT
+
+NVBENCH_BENCH_TYPES(even, NVBENCH_TYPE_AXES(sample_types, counter_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"SampleT{ct}", "CounterT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("Bins", {32, 128, 2048, 2097152})
+  .add_string_axis("Entropy", {"0.201", "1.000"});

cccl_upstream/cub/benchmarks/bench/histogram/histogram_common.cuh

@@ -0,0 +1,78 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#pragma once
+
+#include <cub/device/device_histogram.cuh>
+
+#include <cuda/std/type_traits>
+
+#if !TUNE_BASE
+
+#  if TUNE_LOAD == 0
+#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
+#  elif TUNE_LOAD == 1
+#    define TUNE_LOAD_MODIFIER cub::LOAD_LDG
+#  else // TUNE_LOAD == 2
+#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
+#  endif // TUNE_LOAD
+
+#  define TUNE_VEC_SIZE (1 << TUNE_VEC_SIZE_POW)
+
+#  if TUNE_MEM_PREFERENCE == 0
+constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::GMEM;
+#  elif TUNE_MEM_PREFERENCE == 1
+constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::SMEM;
+#  else // TUNE_MEM_PREFERENCE == 2
+constexpr cub::BlockHistogramMemoryPreference MEM_PREFERENCE = cub::BLEND;
+#  endif // TUNE_MEM_PREFERENCE
+
+#  if TUNE_LOAD_ALGORITHM_ID == 0
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_DIRECT
+#  elif TUNE_LOAD_ALGORITHM_ID == 1
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_WARP_TRANSPOSE
+#  else
+#    define TUNE_LOAD_ALGORITHM cub::BLOCK_LOAD_STRIPED
+#  endif // TUNE_LOAD_ALGORITHM_ID
+
+template <typename SampleT, int NUM_CHANNELS, int NUM_ACTIVE_CHANNELS>
+struct bench_policy_selector
+{
+  _CCCL_API constexpr auto operator()(::cuda::compute_capability) const -> cub::HistogramPolicy
+  {
+    constexpr cub::BlockLoadAlgorithm load_algorithm =
+      (TUNE_LOAD_ALGORITHM == cub::BLOCK_LOAD_STRIPED)
+        ? (NUM_CHANNELS == 1 ? cub::BLOCK_LOAD_STRIPED : cub::BLOCK_LOAD_DIRECT)
+        : TUNE_LOAD_ALGORITHM;
+
+    return {TUNE_THREADS,
+            TUNE_ITEMS,
+            TUNE_VEC_SIZE,
+            load_algorithm,
+            TUNE_LOAD_MODIFIER,
+            TUNE_RLE_COMPRESS,
+            MEM_PREFERENCE,
+            TUNE_WORK_STEALING,
+            2048}; // TODO(bgruber): make tunable
+  }
+};
+#endif // !TUNE_BASE
+
+template <class SampleT, class OffsetT>
+SampleT get_upper_level(OffsetT bins, OffsetT elements)
+{
+  if constexpr (cuda::std::is_integral_v<SampleT>)
+  {
+    if constexpr (sizeof(SampleT) < sizeof(OffsetT))
+    {
+      const SampleT max_key = ::cuda::std::numeric_limits<SampleT>::max();
+      return static_cast<SampleT>(std::min(bins, static_cast<OffsetT>(max_key)));
+    }
+    else
+    {
+      return static_cast<SampleT>(bins);
+    }
+  }
+
+  return static_cast<SampleT>(elements);
+}

cccl_upstream/cub/benchmarks/bench/histogram/multi/even.cu

@@ -0,0 +1,98 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <nvbench_helper.cuh>
+
+#include "../histogram_common.cuh"
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_RLE_COMPRESS rle 0:1:1
+// %RANGE% TUNE_WORK_STEALING ws 0:1:1
+// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1
+// %RANGE% TUNE_LOAD ld 0:2:1
+// %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1
+// %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1
+
+template <typename SampleT, typename CounterT, typename OffsetT>
+static void even(nvbench::state& state, nvbench::type_list<SampleT, CounterT, OffsetT>)
+{
+  constexpr int num_channels        = 4;
+  constexpr int num_active_channels = 3;
+
+  const auto entropy     = str_to_entropy(state.get_string("Entropy"));
+  const auto elements    = state.get_int64("Elements{io}");
+  const auto num_bins    = state.get_int64("Bins");
+  const int num_levels_r = static_cast<int>(num_bins) + 1;
+  const int num_levels_g = num_levels_r;
+  const int num_levels_b = num_levels_g;
+
+  // Skip invalid configurations where LevelT (= SampleT) cannot represent the number of bins
+  if constexpr (cuda::std::is_integral_v<SampleT>)
+  {
+    if (num_bins > static_cast<int64_t>(cuda::std::numeric_limits<SampleT>::max()))
+    {
+      state.skip("Number of bins exceeds what LevelT (= SampleT) can represent");
+      return;
+    }
+  }
+
+  const SampleT lower_level_r = 0;
+  const SampleT upper_level_r = get_upper_level<SampleT>(num_bins, elements);
+  const SampleT lower_level_g = lower_level_r;
+  const SampleT upper_level_g = upper_level_r;
+  const SampleT lower_level_b = lower_level_g;
+  const SampleT upper_level_b = upper_level_g;
+
+  thrust::device_vector<CounterT> hist_r(num_bins);
+  thrust::device_vector<CounterT> hist_g(num_bins);
+  thrust::device_vector<CounterT> hist_b(num_bins);
+  thrust::device_vector<SampleT> input = generate(elements * num_channels, entropy, lower_level_r, upper_level_r);
+
+  SampleT* d_input        = thrust::raw_pointer_cast(input.data());
+  CounterT* d_histogram_r = thrust::raw_pointer_cast(hist_r.data());
+  CounterT* d_histogram_g = thrust::raw_pointer_cast(hist_g.data());
+  CounterT* d_histogram_b = thrust::raw_pointer_cast(hist_b.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<SampleT>(elements * num_active_channels);
+  state.add_global_memory_writes<CounterT>(num_bins * num_active_channels);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<key_t, num_channels, num_active_channels>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      (cub::DeviceHistogram::MultiHistogramEven<num_channels, num_active_channels>),
+      "MultiHistogramEven failed",
+      d_input,
+      cuda::std::array<CounterT*, num_active_channels>{d_histogram_r, d_histogram_g, d_histogram_b},
+      cuda::std::array<int, num_active_channels>{num_levels_r, num_levels_g, num_levels_b},
+      cuda::std::array<SampleT, num_active_channels>{lower_level_r, lower_level_g, lower_level_b},
+      cuda::std::array<SampleT, num_active_channels>{upper_level_r, upper_level_g, upper_level_b},
+      static_cast<OffsetT>(elements),
+      env);
+  });
+}
+
+using counter_types     = nvbench::type_list<int32_t>;
+using some_offset_types = nvbench::type_list<int32_t>;
+
+#ifdef TUNE_SampleT
+using sample_types = nvbench::type_list<TUNE_SampleT>;
+#else // !defined(TUNE_SampleT)
+using sample_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double>;
+#endif // TUNE_SampleT
+
+NVBENCH_BENCH_TYPES(even, NVBENCH_TYPE_AXES(sample_types, counter_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"SampleT{ct}", "CounterT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("Bins", {32, 128, 2048, 2097152})
+  .add_string_axis("Entropy", {"0.201", "1.000"});

cccl_upstream/cub/benchmarks/bench/histogram/multi/range.cu

@@ -0,0 +1,97 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/sequence.h>
+
+#include <nvbench_helper.cuh>
+
+#include "../histogram_common.cuh"
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_RLE_COMPRESS rle 0:1:1
+// %RANGE% TUNE_WORK_STEALING ws 0:1:1
+// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1
+// %RANGE% TUNE_LOAD ld 0:2:1
+// %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1
+// %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1
+
+template <typename SampleT, typename CounterT, typename OffsetT>
+static void range(nvbench::state& state, nvbench::type_list<SampleT, CounterT, OffsetT>)
+{
+  constexpr int num_channels        = 4;
+  constexpr int num_active_channels = 3;
+
+  const auto entropy     = str_to_entropy(state.get_string("Entropy"));
+  const auto elements    = state.get_int64("Elements{io}");
+  const auto num_bins    = state.get_int64("Bins");
+  const int num_levels_r = static_cast<int>(num_bins) + 1;
+  const int num_levels_g = num_levels_r;
+  const int num_levels_b = num_levels_g;
+
+  const SampleT lower_level = 0;
+  const SampleT upper_level = get_upper_level<SampleT>(num_bins, elements);
+
+  SampleT step = (upper_level - lower_level) / num_bins;
+  thrust::device_vector<SampleT> levels_r(num_bins + 1);
+
+  // TODO Extract sequence to the helper TU
+  thrust::sequence(levels_r.begin(), levels_r.end(), lower_level, step);
+  thrust::device_vector<SampleT> levels_g = levels_r;
+  thrust::device_vector<SampleT> levels_b = levels_g;
+
+  SampleT* d_levels_r = thrust::raw_pointer_cast(levels_r.data());
+  SampleT* d_levels_g = thrust::raw_pointer_cast(levels_g.data());
+  SampleT* d_levels_b = thrust::raw_pointer_cast(levels_b.data());
+
+  thrust::device_vector<CounterT> hist_r(num_bins);
+  thrust::device_vector<CounterT> hist_g(num_bins);
+  thrust::device_vector<CounterT> hist_b(num_bins);
+  thrust::device_vector<SampleT> input = generate(elements * num_channels, entropy, lower_level, upper_level);
+
+  SampleT* d_input        = thrust::raw_pointer_cast(input.data());
+  CounterT* d_histogram_r = thrust::raw_pointer_cast(hist_r.data());
+  CounterT* d_histogram_g = thrust::raw_pointer_cast(hist_g.data());
+  CounterT* d_histogram_b = thrust::raw_pointer_cast(hist_b.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<SampleT>(elements * num_active_channels);
+  state.add_global_memory_writes<CounterT>(num_bins * num_active_channels);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<key_t, num_channels, num_active_channels>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      (cub::DeviceHistogram::MultiHistogramRange<num_channels, num_active_channels>),
+      "MultiHistogramRange failed",
+      d_input,
+      cuda::std::array<CounterT*, num_active_channels>{d_histogram_r, d_histogram_g, d_histogram_b},
+      cuda::std::array<int, num_active_channels>{num_levels_r, num_levels_g, num_levels_b},
+      cuda::std::array<const SampleT*, num_active_channels>{d_levels_r, d_levels_g, d_levels_b},
+      static_cast<OffsetT>(elements),
+      env);
+  });
+}
+
+using counter_types     = nvbench::type_list<int32_t>;
+using some_offset_types = nvbench::type_list<int32_t>;
+
+#ifdef TUNE_SampleT
+using sample_types = nvbench::type_list<TUNE_SampleT>;
+#else // !defined(TUNE_SampleT)
+using sample_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double>;
+#endif // TUNE_SampleT
+
+NVBENCH_BENCH_TYPES(range, NVBENCH_TYPE_AXES(sample_types, counter_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"SampleT{ct}", "CounterT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("Bins", {32, 128, 2048, 2097152})
+  .add_string_axis("Entropy", {"0.201", "1.000"});

cccl_upstream/cub/benchmarks/bench/histogram/range.cu

@@ -0,0 +1,83 @@
+// SPDX-FileCopyrightText: Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/sequence.h>
+
+#include <nvbench_helper.cuh>
+
+#include "histogram_common.cuh"
+
+// %RANGE% TUNE_ITEMS ipt 7:24:1
+// %RANGE% TUNE_THREADS tpb 128:1024:32
+// %RANGE% TUNE_RLE_COMPRESS rle 0:1:1
+// %RANGE% TUNE_WORK_STEALING ws 0:1:1
+// %RANGE% TUNE_MEM_PREFERENCE mem 0:2:1
+// %RANGE% TUNE_LOAD ld 0:2:1
+// %RANGE% TUNE_LOAD_ALGORITHM_ID laid 0:2:1
+// %RANGE% TUNE_VEC_SIZE_POW vec 0:2:1
+
+template <typename SampleT, typename CounterT, typename OffsetT>
+static void range(nvbench::state& state, nvbench::type_list<SampleT, CounterT, OffsetT>)
+{
+  const auto entropy   = str_to_entropy(state.get_string("Entropy"));
+  const auto elements  = state.get_int64("Elements{io}");
+  const auto num_bins  = state.get_int64("Bins");
+  const int num_levels = static_cast<int>(num_bins) + 1;
+
+  const SampleT lower_level = 0;
+  const SampleT upper_level = get_upper_level<SampleT>(num_bins, elements);
+
+  SampleT step = (upper_level - lower_level) / num_bins;
+  thrust::device_vector<SampleT> levels(num_bins + 1);
+
+  // TODO Extract sequence to the helper TU
+  thrust::sequence(levels.begin(), levels.end(), lower_level, step);
+  SampleT* d_levels = thrust::raw_pointer_cast(levels.data());
+
+  thrust::device_vector<SampleT> input = generate(elements, entropy, lower_level, upper_level);
+  thrust::device_vector<CounterT> hist(num_bins);
+
+  SampleT* d_input      = thrust::raw_pointer_cast(input.data());
+  CounterT* d_histogram = thrust::raw_pointer_cast(hist.data());
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<SampleT>(elements);
+  state.add_global_memory_writes<CounterT>(num_bins);
+
+  caching_allocator_t alloc;
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
+    auto env = cub_bench_env(
+      alloc,
+      launch
+#if !TUNE_BASE
+      ,
+      cuda::execution::tune(bench_policy_selector<key_t, 1, 1>{})
+#endif // !TUNE_BASE
+    );
+    _CCCL_TRY_CUDA_API(
+      cub::DeviceHistogram::HistogramRange,
+      "HistogramRange failed",
+      d_input,
+      d_histogram,
+      num_levels,
+      d_levels,
+      static_cast<OffsetT>(elements),
+      env);
+  });
+}
+
+using counter_types     = nvbench::type_list<int32_t>;
+using some_offset_types = nvbench::type_list<int32_t>;
+
+#ifdef TUNE_SampleT
+using sample_types = nvbench::type_list<TUNE_SampleT>;
+#else // !defined(TUNE_SampleT)
+using sample_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double>;
+#endif // TUNE_SampleT
+
+NVBENCH_BENCH_TYPES(range, NVBENCH_TYPE_AXES(sample_types, counter_types, some_offset_types))
+  .set_name("base")
+  .set_type_axes_names({"SampleT{ct}", "CounterT{ct}", "OffsetT{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4))
+  .add_int64_axis("Bins", {32, 128, 2048, 2097152})
+  .add_string_axis("Entropy", {"0.201", "1.000"});