[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/cudax/test/coop/any_of/this_thread.cu

@@ -0,0 +1,139 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <cuda/devices>
+#include <cuda/hierarchy>
+#include <cuda/launch>
+#include <cuda/stream>
+
+#include <cuda/experimental/coop.cuh>
+#include <cuda/experimental/group.cuh>
+
+#include "testing.cuh"
+
+template <class Group>
+__device__ void test_group(const Group& group)
+{
+  // Single value tests.
+  {
+    // Test all threads with false.
+    {
+      const auto result = cudax::coop::any_of(group, false);
+      REQUIRE(result.has_value());
+      if (result.has_value())
+      {
+        REQUIRE(!result.value());
+      }
+    }
+
+    // Test all threads with true.
+    {
+      const auto result = cudax::coop::any_of(group, true);
+      REQUIRE(result.has_value());
+      if (result.has_value())
+      {
+        REQUIRE(result.value());
+      }
+    }
+  }
+
+  {
+    // Test all threads with false (broadcasted).
+    {
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, false);
+      REQUIRE(!result);
+    }
+
+    // Test all threads with true (broadcasted).
+    {
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, true);
+      REQUIRE(result);
+    }
+  }
+
+  // Array tests.
+  {
+    // Test all threads with false.
+    {
+      bool in[]{false, false, false};
+      const auto result = cudax::coop::any_of(group, in);
+      REQUIRE(result.has_value());
+      if (result.has_value())
+      {
+        REQUIRE(!result.value());
+      }
+    }
+
+    // Test all threads with true.
+    {
+      bool in[]{true, true, true, true};
+      const auto result = cudax::coop::any_of(group, in);
+      REQUIRE(result.has_value());
+      if (result.has_value())
+      {
+        REQUIRE(result.value());
+      }
+    }
+
+    // Test all threads except 1 with false.
+    {
+      bool in[]{false, false, true};
+      const auto result = cudax::coop::any_of(group, in);
+      REQUIRE(result.has_value());
+      if (result.has_value())
+      {
+        REQUIRE(result.value());
+      }
+    }
+
+    // Test all threads with false (broadcasted).
+    {
+      bool in[]{false, false, false};
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
+      REQUIRE(!result);
+    }
+
+    // Test all threads with true (broadcasted).
+    {
+      bool in[]{true, true, true, true};
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
+      REQUIRE(result);
+    }
+
+    // Test all threads except 1 with false (broadcasted).
+    {
+      bool in[]{false, false, true};
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
+      REQUIRE(result);
+    }
+  }
+}
+
+struct TestKernel
+{
+  template <class Config>
+  __device__ void operator()(const Config& config)
+  {
+    test_group(cudax::this_thread{});
+    test_group(cudax::this_thread{config});
+  }
+};
+
+C2H_TEST("any_of/this_thread", "[any_of][this_thread]")
+{
+  const auto device = cuda::devices[0];
+
+  const cuda::stream stream{device};
+
+  const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
+  cuda::launch(stream, config, TestKernel{});
+
+  stream.sync();
+}

cccl_upstream/cudax/test/coop/any_of/this_warp.cu

@@ -0,0 +1,156 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <cuda/devices>
+#include <cuda/hierarchy>
+#include <cuda/launch>
+#include <cuda/stream>
+
+#include <cuda/experimental/coop.cuh>
+#include <cuda/experimental/group.cuh>
+
+#include "testing.cuh"
+
+template <class Group>
+__device__ void test_group(const Group& group)
+{
+  const auto my_rank = cuda::gpu_thread.rank_as<unsigned>(group);
+
+  // Single value tests.
+  {
+    // Test all threads with false.
+    {
+      const auto result = cudax::coop::any_of(group, false);
+      REQUIRE(result.has_value() == (my_rank == 0));
+      if (result.has_value())
+      {
+        REQUIRE(!result.value());
+      }
+    }
+
+    // Test all threads with true.
+    {
+      const auto result = cudax::coop::any_of(group, true);
+      REQUIRE(result.has_value() == (my_rank == 0));
+      if (result.has_value())
+      {
+        REQUIRE(result.value());
+      }
+    }
+
+    // Test all threads except 1 with false.
+    {
+      const auto result = cudax::coop::any_of(group, (my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1));
+      REQUIRE(result.has_value() == (my_rank == 0));
+      if (result.has_value())
+      {
+        REQUIRE(result.value());
+      }
+    }
+
+    // Test all threads with false (broadcasted).
+    {
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, false);
+      REQUIRE(!result);
+    }
+
+    // Test all threads with true (broadcasted).
+    {
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, true);
+      REQUIRE(result);
+    }
+
+    // Test all threads except 1 with false (broadcasted).
+    {
+      const bool result =
+        cudax::coop::any_of(cudax::broadcasted, group, (my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1));
+      REQUIRE(result);
+    }
+  }
+
+  // Array tests.
+  {
+    // Test all threads with false.
+    {
+      bool in[]{false, false, false};
+      const auto result = cudax::coop::any_of(group, in);
+      REQUIRE(result.has_value() == (my_rank == 0));
+      if (result.has_value())
+      {
+        REQUIRE(!result.value());
+      }
+    }
+
+    // Test all threads with true.
+    {
+      bool in[]{true, true, true, true};
+      const auto result = cudax::coop::any_of(group, in);
+      REQUIRE(result.has_value() == (my_rank == 0));
+      if (result.has_value())
+      {
+        REQUIRE(result.value());
+      }
+    }
+
+    // Test all threads except 1 with false.
+    {
+      bool in[]{false, false, my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1};
+      const auto result = cudax::coop::any_of(group, in);
+      REQUIRE(result.has_value() == (my_rank == 0));
+      if (result.has_value())
+      {
+        REQUIRE(result.value());
+      }
+    }
+
+    // Test all threads with false (broadcasted).
+    {
+      bool in[]{false, false, false};
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
+      REQUIRE(!result);
+    }
+
+    // Test all threads with true (broadcasted).
+    {
+      bool in[]{true, true, true, true};
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
+      REQUIRE(result);
+    }
+
+    // Test all threads except 1 with false (broadcasted).
+    {
+      bool in[]{false, false, my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1};
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
+      REQUIRE(result);
+    }
+  }
+}
+
+struct TestKernel
+{
+  template <class Config>
+  __device__ void operator()(const Config& config)
+  {
+    test_group(cudax::this_warp{});
+    test_group(cudax::this_warp{config});
+  }
+};
+
+C2H_TEST("any_of/this_warp", "[any_of][this_warp]")
+{
+  const auto device = cuda::devices[0];
+
+  const cuda::stream stream{device};
+
+  const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
+  cuda::launch(stream, config, TestKernel{});
+
+  stream.sync();
+}

cccl_upstream/cudax/test/coop/any_of/threads_within_warp.cu

@@ -0,0 +1,187 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <cuda/devices>
+#include <cuda/hierarchy>
+#include <cuda/launch>
+#include <cuda/stream>
+
+#include <cuda/experimental/coop.cuh>
+#include <cuda/experimental/group.cuh>
+
+#include "testing.cuh"
+
+template <class Group>
+__device__ void test_group(const Group& group)
+{
+  // Exit all threads that are not part of the group.
+  if (!cuda::gpu_thread.is_part_of(group))
+  {
+    return;
+  }
+
+  const auto my_rank = cuda::gpu_thread.rank_as<unsigned>(group);
+
+  // Single value tests.
+  {
+    // Test all threads with false.
+    {
+      const auto result = cudax::coop::any_of(group, false);
+      REQUIRE(result.has_value() == (my_rank == 0));
+      if (result.has_value())
+      {
+        REQUIRE(!result.value());
+      }
+    }
+
+    // Test all threads with true.
+    {
+      const auto result = cudax::coop::any_of(group, true);
+      REQUIRE(result.has_value() == (my_rank == 0));
+      if (result.has_value())
+      {
+        REQUIRE(result.value());
+      }
+    }
+
+    // Test all threads except 1 with false.
+    {
+      const auto result = cudax::coop::any_of(group, (my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1));
+      REQUIRE(result.has_value() == (my_rank == 0));
+      if (result.has_value())
+      {
+        REQUIRE(result.value());
+      }
+    }
+
+    // Test all threads with false (broadcasted).
+    {
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, false);
+      REQUIRE(!result);
+    }
+
+    // Test all threads with true (broadcasted).
+    {
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, true);
+      REQUIRE(result);
+    }
+
+    // Test all threads except 1 with false (broadcasted).
+    {
+      const bool result =
+        cudax::coop::any_of(cudax::broadcasted, group, (my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1));
+      REQUIRE(result);
+    }
+  }
+
+  // Array tests.
+  {
+    // Test all threads with false.
+    {
+      bool in[]{false, false, false};
+      const auto result = cudax::coop::any_of(group, in);
+      REQUIRE(result.has_value() == (my_rank == 0));
+      if (result.has_value())
+      {
+        REQUIRE(!result.value());
+      }
+    }
+
+    // Test all threads with true.
+    {
+      bool in[]{true, true, true, true};
+      const auto result = cudax::coop::any_of(group, in);
+      REQUIRE(result.has_value() == (my_rank == 0));
+      if (result.has_value())
+      {
+        REQUIRE(result.value());
+      }
+    }
+
+    // Test all threads except 1 with false.
+    {
+      bool in[]{false, false, my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1};
+      const auto result = cudax::coop::any_of(group, in);
+      REQUIRE(result.has_value() == (my_rank == 0));
+      if (result.has_value())
+      {
+        REQUIRE(result.value());
+      }
+    }
+
+    // Test all threads with false (broadcasted).
+    {
+      bool in[]{false, false, false};
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
+      REQUIRE(!result);
+    }
+
+    // Test all threads with true (broadcasted).
+    {
+      bool in[]{true, true, true, true};
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
+      REQUIRE(result);
+    }
+
+    // Test all threads except 1 with false (broadcasted).
+    {
+      bool in[]{false, false, my_rank == cuda::gpu_thread.count_as<unsigned>(group) - 1};
+      const bool result = cudax::coop::any_of(cudax::broadcasted, group, in);
+      REQUIRE(result);
+    }
+  }
+}
+
+struct CustomBinaryPartition
+{
+  template <class MappingResult>
+  __device__ bool operator()(MappingResult mapping_result)
+  {
+    switch (mapping_result.unit_rank())
+    {
+      case 1:
+      case 5:
+      case 14:
+      case 15:
+      case 31:
+        return true;
+      default:
+        return false;
+    }
+  }
+};
+
+struct TestKernel
+{
+  template <class Config>
+  __device__ void operator()(const Config& config)
+  {
+    const cudax::this_warp warp{config};
+    test_group(cudax::group{cuda::gpu_thread, warp, cudax::identity_mapping{}, cudax::lane_synchronizer{}});
+    test_group(cudax::group{cuda::gpu_thread, warp, cudax::group_by<4>{}, cudax::lane_synchronizer{}});
+    test_group(cudax::group{cuda::gpu_thread, warp, cudax::group_by{1}, cudax::lane_synchronizer{}});
+    test_group(
+      cudax::group{cuda::gpu_thread, warp, cudax::group_by{3, cudax::non_exhaustive}, cudax::lane_synchronizer{}});
+    test_group(cudax::group{
+      cuda::gpu_thread, warp, cudax::binary_partition{CustomBinaryPartition{}}, cudax::lane_synchronizer{}});
+  }
+};
+
+C2H_TEST("any_of/threads_within_warp", "[any_of][threads_within_warp]")
+{
+  const auto device = cuda::devices[0];
+
+  const cuda::stream stream{device};
+
+  const auto config = cuda::make_config(cuda::grid_dims<1>(), cuda::block_dims<32>());
+  cuda::launch(stream, config, TestKernel{});
+
+  stream.sync();
+}