[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/thrust/benchmarks/bench/transform/babelstream.cu

@@ -0,0 +1,173 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/iterator/zip_iterator.h>
+#include <thrust/transform.h>
+#include <thrust/zip_function.h>
+
+#include <cuda/__functional/address_stability.h>
+
+#include <nvbench_helper.cuh>
+
+// The benchmarks are inspired by the BabelStream thrust version:
+// https://github.com/UoB-HPC/BabelStream/blob/main/src/thrust/ThrustStream.cu
+
+// Modified from BabelStream to also work for integers
+constexpr auto startA      = 1; // BabelStream: 0.1
+constexpr auto startB      = 2; // BabelStream: 0.2
+constexpr auto startC      = 3; // BabelStream: 0.1
+constexpr auto startScalar = 4; // BabelStream: 0.4
+
+using element_types = nvbench::type_list<std::int8_t, std::int16_t, float, double, __int128>;
+// Different benchmarks use a different number of buffers. H200/B200 can fit 2^31 elements for all benchmarks and types.
+// Upstream BabelStream uses 2^25. Allocation failure just skips the benchmark
+auto array_size_powers = std::vector<std::int64_t>{25, 31};
+
+template <typename... Args>
+void bench_transform(nvbench::state& state, Args&&... args)
+{
+  caching_allocator_t alloc; // transform shouldn't allocate, but let's be consistent
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::transform(policy(alloc, launch), ::cuda::std::forward<Args>(args)...);
+             });
+}
+
+template <typename T>
+static void mul(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> b(n, startB);
+  thrust::device_vector<T> c(n, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(n);
+
+  const T scalar = startScalar;
+  bench_transform(
+    state, c.begin(), c.end(), b.begin(), cuda::proclaim_copyable_arguments([=] _CCCL_DEVICE(const T& ci) {
+      return ci * scalar;
+    }));
+}
+
+NVBENCH_BENCH_TYPES(mul, NVBENCH_TYPE_AXES(element_types))
+  .set_name("mul")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", array_size_powers);
+
+template <typename T>
+static void add(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> a(n, startA);
+  thrust::device_vector<T> b(n, startB);
+  thrust::device_vector<T> c(n, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+
+  bench_transform(
+    state,
+    a.begin(),
+    a.end(),
+    b.begin(),
+    c.begin(),
+    cuda::proclaim_copyable_arguments([] _CCCL_DEVICE(const T& ai, const T& bi) -> T {
+      return ai + bi;
+    }));
+}
+
+NVBENCH_BENCH_TYPES(add, NVBENCH_TYPE_AXES(element_types))
+  .set_name("add")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", array_size_powers);
+
+template <typename T>
+static void triad(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> a(n, startA);
+  thrust::device_vector<T> b(n, startB);
+  thrust::device_vector<T> c(n, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+
+  const T scalar = startScalar;
+  bench_transform(
+    state,
+    b.begin(),
+    b.end(),
+    c.begin(),
+    a.begin(),
+    cuda::proclaim_copyable_arguments([=] _CCCL_DEVICE(const T& bi, const T& ci) {
+      return bi + scalar * ci;
+    }));
+}
+
+NVBENCH_BENCH_TYPES(triad, NVBENCH_TYPE_AXES(element_types))
+  .set_name("triad")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", array_size_powers);
+
+template <typename T>
+static void nstream(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> a(n, startA);
+  thrust::device_vector<T> b(n, startB);
+  thrust::device_vector<T> c(n, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(3 * n);
+  state.add_global_memory_writes<T>(n);
+
+  const T scalar = startScalar;
+  bench_transform(
+    state,
+    thrust::make_zip_iterator(a.begin(), b.begin(), c.begin()),
+    thrust::make_zip_iterator(a.end(), b.end(), c.end()),
+    a.begin(),
+    thrust::make_zip_function(cuda::proclaim_copyable_arguments([=] _CCCL_DEVICE(const T& ai, const T& bi, const T& ci) {
+      return ai + bi + scalar * ci;
+    })));
+}
+
+NVBENCH_BENCH_TYPES(nstream, NVBENCH_TYPE_AXES(element_types))
+  .set_name("nstream")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", array_size_powers);
+
+// variation of nstream requiring a stable parameter address because it recovers the element index
+template <typename T>
+static void nstream_stable(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> a(n, startA);
+  thrust::device_vector<T> b(n, startB);
+  thrust::device_vector<T> c(n, startC);
+
+  const T* a_start = thrust::raw_pointer_cast(a.data());
+  const T* b_start = thrust::raw_pointer_cast(b.data());
+  const T* c_start = thrust::raw_pointer_cast(c.data());
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(3 * n);
+  state.add_global_memory_writes<T>(n);
+
+  const T scalar = startScalar;
+  bench_transform(state, a.begin(), a.end(), a.begin(), [=] _CCCL_DEVICE(const T& ai) {
+    const auto i = &ai - a_start;
+    return ai + b_start[i] + scalar * c_start[i];
+  });
+}
+
+NVBENCH_BENCH_TYPES(nstream_stable, NVBENCH_TYPE_AXES(element_types))
+  .set_name("nstream_stable")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", array_size_powers);

cccl_upstream/thrust/benchmarks/bench/transform/fib.cu

@@ -0,0 +1,62 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/transform.h>
+
+#include <nvbench_helper.cuh>
+
+template <class InT, class OutT>
+struct fib_t
+{
+  __device__ OutT operator()(InT n)
+  {
+    OutT t1 = 0;
+    OutT t2 = 1;
+
+    if (n <= 1)
+    {
+      return t1;
+    }
+    else if (n == 2)
+    {
+      return t2;
+    }
+    for (InT i = 3; i <= n; ++i)
+    {
+      const auto next = t1 + t2;
+      t1              = t2;
+      t2              = next;
+    }
+
+    return t2;
+  }
+};
+
+template <typename T>
+static void fib(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto elements = static_cast<std::size_t>(state.get_int64("Elements"));
+
+  thrust::device_vector<T> input = generate(elements, bit_entropy::_1_000, T{0}, T{42});
+  thrust::device_vector<T> output(elements);
+
+  state.add_element_count(elements);
+  state.add_global_memory_reads<T>(elements);
+  state.add_global_memory_writes<nvbench::uint32_t>(elements);
+
+  fib_t<T, nvbench::uint32_t> op{};
+  caching_allocator_t alloc; // transform shouldn't allocate, but let's be consistent
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::transform(policy(alloc, launch), input.cbegin(), input.cend(), output.begin(), op);
+             });
+}
+
+using types = nvbench::type_list<nvbench::uint32_t, nvbench::uint64_t>;
+
+NVBENCH_BENCH_TYPES(fib, NVBENCH_TYPE_AXES(types))
+  .set_name("fib")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4));

cccl_upstream/thrust/benchmarks/bench/transform/if.cu

@@ -0,0 +1,38 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#include <thrust/count.h>
+#include <thrust/device_vector.h>
+#include <thrust/transform.h>
+
+#include <nvbench_helper.cuh>
+
+template <typename T>
+static void negate_if(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto entropy = str_to_entropy(state.get_string("Entropy"));
+  const auto val     = lerp_min_max<T>(entropy_to_probability(entropy));
+  auto transform_op  = ::cuda::std::negate<T>{};
+  auto select_op     = less_then_t<T>{val};
+
+  thrust::device_vector<T> input = generate(n);
+  thrust::device_vector<T> output(n, thrust::no_init);
+
+  const auto selected_elements = thrust::count_if(input.cbegin(), input.cend(), select_op);
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(selected_elements);
+
+  caching_allocator_t alloc; // transform_if shouldn't allocate, but let's be consistent
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      thrust::transform_if(policy(alloc, launch), input.begin(), input.end(), output.begin(), transform_op, select_op);
+    });
+}
+
+NVBENCH_BENCH_TYPES(negate_if, NVBENCH_TYPE_AXES(integral_types))
+  //  .set_name("negate_if")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.000"});

cccl_upstream/thrust/benchmarks/bench/transform/if_stencil.cu

@@ -0,0 +1,39 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/count.h>
+#include <thrust/device_vector.h>
+#include <thrust/transform.h>
+
+#include <nvbench_helper.cuh>
+
+template <typename T>
+static void negate_if_stencil(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n       = static_cast<std::size_t>(state.get_int64("Elements"));
+  const auto entropy = str_to_entropy(state.get_string("Entropy"));
+  const auto val     = lerp_min_max<T>(entropy_to_probability(entropy));
+  auto transform_op  = ::cuda::std::negate<T>{};
+  auto select_op     = less_then_t<T>{val};
+
+  thrust::device_vector<T> input   = generate(n);
+  thrust::device_vector<T> stencil = generate(n);
+  thrust::device_vector<T> output(n, thrust::no_init);
+
+  const auto selected_elements = thrust::count_if(input.cbegin(), input.cend(), select_op);
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n + selected_elements);
+  state.add_global_memory_writes<T>(selected_elements);
+
+  caching_allocator_t alloc; // transform_if shouldn't allocate, but let's be consistent
+  state.exec(
+    nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync, [&](nvbench::launch& launch) {
+      thrust::transform_if(
+        policy(alloc, launch), input.begin(), input.end(), stencil.begin(), output.begin(), transform_op, select_op);
+    });
+}
+
+NVBENCH_BENCH_TYPES(negate_if_stencil, NVBENCH_TYPE_AXES(integral_types))
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", nvbench::range(16, 28, 4))
+  .add_string_axis("Entropy", {"1.000", "0.544", "0.000"});

cccl_upstream/thrust/benchmarks/bench/transform/zip_transform.cu

@@ -0,0 +1,55 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+
+#include <cuda/functional>
+#include <cuda/iterator>
+
+#include <nvbench_helper.cuh>
+
+// This benchmark is intended to be compared to the nstream benchmark from babelstream.cu, so we can:
+// * detect regressions in the unpacking of a zip_transform_iterator
+
+// same variables as in basic.cu so we can compare results
+constexpr auto startA      = 1; // BabelStream: 0.1
+constexpr auto startB      = 2; // BabelStream: 0.2
+constexpr auto startC      = 3; // BabelStream: 0.1
+constexpr auto startScalar = 4; // BabelStream: 0.4
+using element_types        = nvbench::type_list<std::int8_t, std::int16_t, float, double, __int128>;
+auto array_size_powers     = std::vector<std::int64_t>{25, 31};
+
+template <typename T>
+static void nstream_zip_transform(nvbench::state& state, nvbench::type_list<T>)
+{
+  const auto n = static_cast<std::size_t>(state.get_int64("Elements"));
+  thrust::device_vector<T> a(n, startA);
+  thrust::device_vector<T> b(n, startB);
+  thrust::device_vector<T> c(n, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(3 * n);
+  state.add_global_memory_writes<T>(n);
+
+  const T scalar = startScalar;
+  auto lambda    = cuda::proclaim_copyable_arguments([scalar] _CCCL_DEVICE(const T& ai, const T& bi, const T& ci) -> T {
+    // Needed to silence clangs -Wunused-lambda-capture. We cannot just remove it because other
+    // implementations (e.g. MSVC) will emit errors if we don't capture it. See discussion in
+    // https://reviews.llvm.org/D28467.
+    static_cast<void>(scalar);
+    return ai + bi + scalar * ci;
+  });
+  cuda::zip_transform_iterator begin{lambda, a.begin(), b.begin(), c.begin()};
+  cuda::zip_transform_iterator end{lambda, a.end(), b.end(), c.end()};
+  caching_allocator_t alloc; // transform shouldn't allocate, but let's be consistent
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch | nvbench::exec_tag::sync,
+             [&](nvbench::launch& launch) {
+               thrust::copy(policy(alloc, launch), begin, end, a.begin());
+             });
+}
+
+NVBENCH_BENCH_TYPES(nstream_zip_transform, NVBENCH_TYPE_AXES(element_types))
+  .set_name("nstream")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements", array_size_powers);