[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/thrust/examples/cuda/CMakeLists.txt

@@ -0,0 +1,20 @@
+file(
+  GLOB example_srcs
+  RELATIVE "${CMAKE_CURRENT_LIST_DIR}"
+  CONFIGURE_DEPENDS
+  *.cu
+  *.cpp
+)
+
+foreach (thrust_target IN LISTS THRUST_TARGETS)
+  thrust_get_target_property(config_device ${thrust_target} DEVICE)
+  if (NOT config_device STREQUAL "CUDA")
+    continue()
+  endif()
+
+  foreach (example_src IN LISTS example_srcs)
+    get_filename_component(example_name "${example_src}" NAME_WLE)
+    string(PREPEND example_name "cuda.")
+    thrust_add_example(example_target ${example_name} "${example_src}" ${thrust_target})
+  endforeach()
+endforeach()

cccl_upstream/thrust/examples/cuda/async_reduce.cu

@@ -0,0 +1,78 @@
+#include <thrust/detail/config.h>
+
+#include <thrust/device_vector.h>
+#include <thrust/reduce.h>
+#include <thrust/system/cuda/execution_policy.h>
+
+#include <cassert>
+#include <future>
+
+// This example demonstrates two ways to achieve algorithm invocations that are asynchronous with
+// the calling thread.
+//
+// The first method wraps a call to thrust::reduce inside a __global__ function. Since __global__ function
+// launches are asynchronous with the launching thread, this achieves asynchrony. The result of the reduction
+// is stored to a pointer to CUDA global memory. The calling thread waits for the result of the reduction to
+// be ready by synchronizing with the CUDA stream on which the __global__ function is launched.
+//
+// The second method uses the C++11 library function, std::async, to create concurrency. The lambda function
+// given to std::async returns the result of thrust::reduce to a std::future. The calling thread can use the
+// std::future to wait for the result of the reduction. This method requires a compiler which supports
+// C++11-capable language and library constructs.
+
+#ifdef THRUST_EXAMPLE_DEVICE_SIDE
+template <typename Iterator, typename T, typename BinaryOperation, typename Pointer>
+__global__ void reduce_kernel(Iterator first, Iterator last, T init, BinaryOperation binary_op, Pointer result)
+{
+  *result = thrust::reduce(thrust::cuda::par, first, last, init, binary_op);
+}
+#endif
+
+int main()
+{
+  size_t n = 1 << 20;
+  thrust::device_vector<unsigned int> data(n, 1);
+  thrust::device_vector<unsigned int> result(1, 0);
+
+  // method 1: call thrust::reduce from an asynchronous CUDA kernel launch
+
+  // create a CUDA stream
+  cudaStream_t s;
+  cudaStreamCreate(&s);
+
+  // launch a CUDA kernel with only 1 thread on our stream
+#ifdef THRUST_EXAMPLE_DEVICE_SIDE
+  reduce_kernel<<<1, 1, 0, s>>>(data.begin(), data.end(), 0, cuda::std::plus<int>(), result.data());
+#else
+  result[0] = thrust::reduce(thrust::cuda::par, data.begin(), data.end(), 0, cuda::std::plus<int>());
+#endif
+
+  // wait for the stream to finish
+  cudaStreamSynchronize(s);
+
+  // our result should be ready
+  assert(result[0] == n);
+
+  cudaStreamDestroy(s);
+
+  // reset the result
+  result[0] = 0;
+
+  // method 2: use std::async to create asynchrony
+  // copy all the algorithm parameters
+  auto begin        = data.begin();
+  auto end          = data.end();
+  unsigned int init = 0;
+  auto binary_op    = cuda::std::plus<unsigned int>();
+
+  // std::async captures the algorithm parameters by value
+  // use std::launch::async to ensure the creation of a new thread
+  std::future<unsigned int> future_result = std::async(std::launch::async, [=] {
+    return thrust::reduce(begin, end, init, binary_op);
+  });
+
+  // wait on the result and check that it is correct
+  assert(future_result.get() == n);
+
+  return 0;
+}

cccl_upstream/thrust/examples/cuda/custom_temporary_allocation.cu

@@ -0,0 +1,154 @@
+#include <thrust/generate.h>
+#include <thrust/host_vector.h>
+#include <thrust/sort.h>
+#include <thrust/system/cuda/execution_policy.h>
+#include <thrust/system/cuda/vector.h>
+
+#include <cuda/std/utility>
+
+#include <cassert>
+#include <cstdlib>
+#include <iostream>
+#include <map>
+#include <sstream>
+#include <stdexcept>
+
+// This example demonstrates how to control how Thrust allocates temporary
+// storage during algorithms such as thrust::sort. The idea will be to create a
+// simple cache of allocations to search when temporary storage is requested.
+// If a hit is found in the cache, we quickly return the cached allocation
+// instead of resorting to the more expensive thrust::cuda::malloc.
+
+// Note: Thrust now has its own caching allocator layer; if you just need a
+// caching allocator, you ought to use that. This example is still useful
+// as a demonstration of how to use a Thrust custom allocator.
+
+// Note: this implementation cached_allocator is not thread-safe. If multiple
+// (host) threads use the same cached_allocator then they should gain exclusive
+// access to the allocator before accessing its methods.
+
+struct not_my_pointer_exception : std::exception
+{
+  explicit not_my_pointer_exception(void* p)
+  {
+    std::stringstream s;
+    s << "Pointer `" << p << "` was not allocated by this allocator.";
+    message = s.str();
+  }
+
+  const char* what() const noexcept override
+  {
+    return message.c_str();
+  }
+
+private:
+  std::string message;
+};
+
+// A simple allocator for caching cudaMalloc allocations.
+// A minimum allocator needs to provide at least `value_type`, `allocate` and `deallocate`.
+struct cached_allocator
+{
+  using value_type = char;
+
+  ~cached_allocator()
+  {
+    free_all();
+  }
+
+  char* allocate(std::ptrdiff_t num_bytes)
+  {
+    std::cout << "cached_allocator::allocate(): num_bytes == " << num_bytes << '\n';
+
+    char* result = nullptr;
+
+    // Search the cache for a free block.
+    auto free_block_it = free_blocks.find(num_bytes);
+    if (free_block_it != free_blocks.end())
+    {
+      std::cout << "cached_allocator::allocate(): found a free block" << '\n';
+      result = free_block_it->second;
+      free_blocks.erase(free_block_it);
+    }
+    else
+    {
+      // No allocation of the right size exists, so create a new one with `thrust::cuda::malloc`.
+      std::cout << "cached_allocator::allocate(): allocating new block" << '\n';
+      // Allocate memory and convert the resulting `thrust::cuda::pointer` to a raw pointer.
+      result = thrust::cuda::malloc<char>(num_bytes).get();
+    }
+
+    // Insert the allocated pointer into the `allocated_blocks` map.
+    allocated_blocks.insert(std::pair{result, num_bytes});
+
+    return result;
+  }
+
+  void deallocate(char* ptr, size_t)
+  {
+    std::cout << "cached_allocator::deallocate(): ptr == " << reinterpret_cast<void*>(ptr) << '\n';
+
+    // Erase the allocated block from the allocated blocks map.
+    auto it = allocated_blocks.find(ptr);
+    if (it == allocated_blocks.end())
+    {
+      throw not_my_pointer_exception(ptr);
+    }
+
+    const std::ptrdiff_t num_bytes = it->second;
+    allocated_blocks.erase(it);
+
+    // Insert the block into the free blocks map.
+    free_blocks.insert(std::make_pair(num_bytes, ptr));
+  }
+
+private:
+  std::multimap<std::ptrdiff_t, char*> free_blocks;
+  std::map<char*, std::ptrdiff_t> allocated_blocks;
+
+  void free_all()
+  {
+    std::cout << "cached_allocator::free_all()" << '\n';
+
+    // Deallocate all outstanding blocks in both lists.
+    for (auto [bytes, ptr] : free_blocks)
+    {
+      thrust::cuda::free(thrust::cuda::pointer<char>(ptr));
+    }
+
+    for (auto [ptr, bytes] : allocated_blocks)
+    {
+      thrust::cuda::free(thrust::cuda::pointer<char>(ptr));
+    }
+  }
+};
+
+int main()
+{
+  std::size_t num_elements = 32768;
+
+  thrust::host_vector<int> h_input(num_elements);
+
+  // Generate random input.
+  thrust::generate(h_input.begin(), h_input.end(), rand);
+
+  thrust::cuda::vector<int> d_input = h_input;
+  thrust::cuda::vector<int> d_result(num_elements);
+
+  std::size_t num_trials = 5;
+
+  cached_allocator alloc;
+
+  for (std::size_t i = 0; i < num_trials; ++i)
+  {
+    d_result = d_input;
+
+    // Pass alloc to execution policy cuda::par. It will handle allocations needed inside sort.
+    thrust::sort(thrust::cuda::par(alloc), d_result.begin(), d_result.end());
+
+    // Ensure the result is sorted.
+    assert(thrust::is_sorted(d_result.begin(), d_result.end()));
+  }
+
+  return 0;
+}

cccl_upstream/thrust/examples/cuda/explicit_cuda_stream.cu

@@ -0,0 +1,77 @@
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h> // For thrust::device
+#include <thrust/reduce.h>
+#include <thrust/sequence.h>
+
+#include <iostream>
+
+#include <cuda_runtime.h>
+
+// This example shows how to execute a Thrust device algorithm on an explicit
+// CUDA stream. The simple program below fills a vector with the numbers
+// [0, 1000) (thrust::sequence) and then performs a scan operation
+// (thrust::inclusive_scan) on them. Both algorithms are executed on the same
+// custom CUDA stream using the CUDA execution policies.
+//
+// Thrust provides two execution policies that accept CUDA streams that differ
+// in when/if they synchronize the stream:
+// 1. thrust::cuda::par.on(stream)
+//      - `stream` will *always* be synchronized before an algorithm returns.
+//      - This is the default `thrust::device` policy when compiling with the
+//        CUDA device backend.
+// 2. thrust::cuda::par_nosync.on(stream)
+//      - `stream` will only be synchronized when necessary for correctness
+//        (e.g., returning a result from `thrust::reduce`). This is a hint that
+//        may be ignored by an algorithm's implementation.
+
+int main()
+{
+  thrust::device_vector<int> d_vec(1000);
+
+  // Create the stream:
+  cudaStream_t custom_stream;
+  cudaError_t err = cudaStreamCreate(&custom_stream);
+  if (err != cudaSuccess)
+  {
+    std::cerr << "Error creating stream: " << cudaGetErrorString(err) << "\n";
+    return 1;
+  }
+
+  // Construct a new `nosync` execution policy with the custom stream
+  auto nosync_exec_policy = thrust::cuda::par_nosync.on(custom_stream);
+
+  // Fill the vector with sequential data.
+  // This will execute using the custom stream and the stream will *not* be
+  // synchronized before the function returns, meaning asynchronous work may
+  // still be executing after returning and the contents of `d_vec` are
+  // undefined. Synchronization is not needed here because the following
+  // `inclusive_scan` is executed on the same stream and is therefore guaranteed
+  // to be ordered after the `sequence`
+  thrust::sequence(nosync_exec_policy, d_vec.begin(), d_vec.end());
+
+  // Construct a new *synchronous* execution policy with the same custom stream
+  auto sync_exec_policy = thrust::cuda::par.on(custom_stream);
+
+  // Compute in-place inclusive sum scan of data in the vector.
+  // This also executes in the custom stream, but the execution policy ensures
+  // the stream is synchronized before the algorithm returns. This guarantees
+  // there is no pending asynchronous work and the contents of `d_vec` are
+  // immediately accessible.
+  thrust::inclusive_scan(sync_exec_policy, d_vec.cbegin(), d_vec.cend(), d_vec.begin());
+
+  // This access is only valid because the stream has been synchronized
+  int sum = d_vec.back();
+
+  // Free the stream:
+  err = cudaStreamDestroy(custom_stream);
+  if (err != cudaSuccess)
+  {
+    std::cerr << "Error destroying stream: " << cudaGetErrorString(err) << "\n";
+    return 1;
+  }
+
+  // Print the sum:
+  std::cout << "sum is " << sum << '\n';
+
+  return 0;
+}

cccl_upstream/thrust/examples/cuda/global_device_vector.cu

@@ -0,0 +1,34 @@
+#include <thrust/detail/config.h>
+
+#include <thrust/device_vector.h>
+
+// If you create a global `thrust::device_vector` with the default allocator,
+// you'll get an error during program termination when the memory of the vector
+// is freed, as the CUDA runtime cannot be used during program termination.
+//
+// To get around this, you can create your own allocator which ignores
+// deallocation failures that occur because the CUDA runtime is shut down.
+
+extern "C" cudaError_t cudaFreeIgnoreShutdown(void* ptr)
+{
+  cudaError_t const err = cudaFree(ptr);
+  if (cudaSuccess == err || cudaErrorCudartUnloading == err)
+  {
+    return cudaSuccess;
+  }
+  return err;
+}
+
+using device_ignore_shutdown_memory_resource =
+  thrust::system::cuda::detail::cuda_memory_resource<cudaMalloc, cudaFreeIgnoreShutdown, thrust::cuda::pointer<void>>;
+
+template <typename T>
+using device_ignore_shutdown_allocator =
+  thrust::mr::stateless_resource_allocator<T, thrust::device_ptr_memory_resource<device_ignore_shutdown_memory_resource>>;
+
+thrust::device_vector<double, device_ignore_shutdown_allocator<double>> d_vec;
+
+int main()
+{
+  d_vec.resize(25);
+}

cccl_upstream/thrust/examples/cuda/range_view.cu

@@ -0,0 +1,209 @@
+#include <thrust/device_vector.h>
+#include <thrust/execution_policy.h>
+#include <thrust/for_each.h>
+#include <thrust/iterator/counting_iterator.h>
+
+#include <cuda/std/iterator>
+
+#include <iostream>
+
+// This example demonstrates the use of a view: a non-owning wrapper for an
+// iterator range which presents a container-like interface to the user.
+//
+// For example, a view of a device_vector's data can be helpful when we wish to
+// access that data from a device function. Even though device_vectors are not
+// accessible from device functions, the range_view class allows us to access
+// and manipulate its data as if we were manipulating a real container.
+
+template <class Iterator>
+class range_view
+{
+public:
+  using iterator        = Iterator;
+  using value_type      = typename cuda::std::iterator_traits<iterator>::value_type;
+  using pointer         = typename cuda::std::iterator_traits<iterator>::pointer;
+  using difference_type = typename cuda::std::iterator_traits<iterator>::difference_type;
+  using reference       = typename cuda::std::iterator_traits<iterator>::reference;
+
+private:
+  const iterator first;
+  const iterator last;
+
+public:
+  __host__ __device__ range_view(Iterator first, Iterator last)
+      : first(first)
+      , last(last)
+  {}
+  ~range_view() = default;
+
+  __host__ __device__ difference_type size() const
+  {
+    return cuda::std::distance(first, last);
+  }
+
+  __host__ __device__ reference operator[](difference_type n)
+  {
+    return *(first + n);
+  }
+  __host__ __device__ const reference operator[](difference_type n) const
+  {
+    return *(first + n);
+  }
+
+  __host__ __device__ iterator begin()
+  {
+    return first;
+  }
+  __host__ __device__ const iterator cbegin() const
+  {
+    return first;
+  }
+  __host__ __device__ iterator end()
+  {
+    return last;
+  }
+  __host__ __device__ const iterator cend() const
+  {
+    return last;
+  }
+
+  __host__ __device__ cuda::std::reverse_iterator<iterator> rbegin()
+  {
+    return cuda::std::reverse_iterator<iterator>(end());
+  }
+  __host__ __device__ const cuda::std::reverse_iterator<const iterator> crbegin() const
+  {
+    return cuda::std::reverse_iterator<const iterator>(cend());
+  }
+  __host__ __device__ cuda::std::reverse_iterator<iterator> rend()
+  {
+    return cuda::std::reverse_iterator<iterator>(begin());
+  }
+  __host__ __device__ const cuda::std::reverse_iterator<const iterator> crend() const
+  {
+    return cuda::std::reverse_iterator<const iterator>(cbegin());
+  }
+  __host__ __device__ reference front()
+  {
+    return *begin();
+  }
+  __host__ __device__ const reference front() const
+  {
+    return *cbegin();
+  }
+
+  __host__ __device__ reference back()
+  {
+    return *end();
+  }
+  __host__ __device__ const reference back() const
+  {
+    return *cend();
+  }
+
+  __host__ __device__ bool empty() const
+  {
+    return size() == 0;
+  }
+};
+
+// This helper function creates a range_view from iterator and the number of
+// elements
+template <class Iterator, class Size>
+range_view<Iterator> __host__ __device__ make_range_view(Iterator first, Size n)
+{
+  return range_view<Iterator>(first, first + n);
+}
+
+// This helper function creates a range_view from a pair of iterators
+template <class Iterator>
+range_view<Iterator> __host__ __device__ make_range_view(Iterator first, Iterator last)
+{
+  return range_view<Iterator>(first, last);
+}
+
+// This helper function creates a range_view from a Vector
+template <class Vector>
+range_view<typename Vector::iterator> __host__ make_range_view(Vector& v)
+{
+  return range_view<typename Vector::iterator>(v.begin(), v.end());
+}
+
+// This saxpy functor stores view of X, Y, Z array, and accesses them in
+// vector-like way
+template <class View1, class View2, class View3>
+struct saxpy_functor
+{
+  const float a;
+  View1 x;
+  View2 y;
+  View3 z;
+
+  __host__ __device__ saxpy_functor(float _a, View1 _x, View2 _y, View3 _z)
+      : a(_a)
+      , x(_x)
+      , y(_y)
+      , z(_z)
+  {}
+
+  __host__ __device__ void operator()(int i)
+  {
+    z[i] = a * x[i] + y[i];
+  }
+};
+
+// saxpy function, which can either be called form host or device
+// The views are passed by value
+template <class View1, class View2, class View3>
+__host__ __device__ void saxpy(float A, View1 X, View2 Y, View3 Z)
+{
+  // Z = A * X + Y
+  const int size = X.size();
+  thrust::for_each(thrust::device,
+                   thrust::make_counting_iterator(0),
+                   thrust::make_counting_iterator(size),
+                   saxpy_functor<View1, View2, View3>(A, X, Y, Z));
+}
+
+struct f1
+{
+  __host__ __device__ float operator()(float x) const
+  {
+    return x * 3;
+  }
+};
+
+int main()
+{
+  using std::cout;
+
+  // initialize host arrays
+  float x[4] = {1.0, 1.0, 1.0, 1.0};
+  float y[4] = {1.0, 2.0, 3.0, 4.0};
+  float z[4] = {0.0};
+
+  thrust::device_vector<float> X(x, x + 4);
+  thrust::device_vector<float> Y(y, y + 4);
+  thrust::device_vector<float> Z(z, z + 4);
+
+  saxpy(
+    2.0,
+
+    // make a range view of a pair of transform_iterators
+    make_range_view(thrust::make_transform_iterator(X.cbegin(), f1()), thrust::make_transform_iterator(X.cend(), f1())),
+
+    // range view of normal_iterators
+    make_range_view(Y.begin(), cuda::std::distance(Y.begin(), Y.end())),
+
+    // range view of naked pointers
+    make_range_view(Z.data().get(), 4));
+
+  // print values from original device_vector<float> Z
+  // to ensure that range view was mapped to this vector
+  for (std::size_t i = 0, n = Z.size(); i < n; ++i)
+  {
+    cout << "z[" << i << "]= " << Z[i] << '\n';
+  }
+
+  return 0;
+}

cccl_upstream/thrust/examples/cuda/unwrap_pointer.cu

@@ -0,0 +1,31 @@
+#include <thrust/device_free.h>
+#include <thrust/device_malloc.h>
+#include <thrust/device_ptr.h>
+#include <thrust/device_vector.h>
+
+#include <cuda.h>
+
+int main()
+{
+  size_t N = 10;
+
+  // create a device_ptr
+  thrust::device_ptr<int> dev_ptr = thrust::device_malloc<int>(N);
+
+  // extract raw pointer from device_ptr
+  int* raw_ptr = thrust::raw_pointer_cast(dev_ptr);
+
+  // use raw_ptr in CUDA API functions
+  cudaMemset(raw_ptr, 0, N * sizeof(int));
+
+  // free memory
+  thrust::device_free(dev_ptr);
+
+  // we can use the same approach for device_vector
+  thrust::device_vector<int> d_vec(N);
+
+  // note: d_vec.data() returns a device_ptr
+  raw_ptr = thrust::raw_pointer_cast(d_vec.data());
+
+  return 0;
+}

cccl_upstream/thrust/examples/cuda/wrap_pointer.cu

@@ -0,0 +1,27 @@
+#include <thrust/device_ptr.h>
+#include <thrust/fill.h>
+
+#include <cuda.h>
+
+int main()
+{
+  size_t N = 10;
+
+  // obtain raw pointer to device memory
+  int* raw_ptr;
+  cudaMalloc((void**) &raw_ptr, N * sizeof(int));
+
+  // wrap raw pointer with a device_ptr
+  thrust::device_ptr<int> dev_ptr = thrust::device_pointer_cast(raw_ptr);
+
+  // use device_ptr in Thrust algorithms
+  thrust::fill(dev_ptr, dev_ptr + static_cast<std::ptrdiff_t>(N), (int) 0);
+
+  // access device memory transparently through device_ptr
+  dev_ptr[0] = 1;
+
+  // free memory
+  cudaFree(raw_ptr);
+
+  return 0;
+}