[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/examples/ccclrt/kernel_launch_patterns/CMakeLists.txt

@@ -0,0 +1,57 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_executable(cccl.examples.ccclrt.kernel_launch_patterns.kernel kernel.cu)
+set_target_properties(
+  cccl.examples.ccclrt.kernel_launch_patterns.kernel
+  PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON
+)
+target_link_libraries(
+  cccl.examples.ccclrt.kernel_launch_patterns.kernel
+  PRIVATE libcudacxx::libcudacxx
+)
+
+add_executable(
+  cccl.examples.ccclrt.kernel_launch_patterns.kernel_functor
+  kernel_functor.cu
+)
+set_target_properties(
+  cccl.examples.ccclrt.kernel_launch_patterns.kernel_functor
+  PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON
+)
+target_link_libraries(
+  cccl.examples.ccclrt.kernel_launch_patterns.kernel_functor
+  PRIVATE libcudacxx::libcudacxx
+)
+
+# Kernel lambdas require extended lambda support.
+if ("${CMAKE_CUDA_COMPILER_ID}" STREQUAL NVIDIA)
+  add_executable(
+    cccl.examples.ccclrt.kernel_launch_patterns.kernel_lambda
+    kernel_lambda.cu
+  )
+  set_target_properties(
+    cccl.examples.ccclrt.kernel_launch_patterns.kernel_lambda
+    PROPERTIES CUDA_STANDARD 17 CUDA_STANDARD_REQUIRED ON
+  )
+  target_compile_options(
+    cccl.examples.ccclrt.kernel_launch_patterns.kernel_lambda
+    PRIVATE -extended-lambda
+  )
+  target_link_libraries(
+    cccl.examples.ccclrt.kernel_launch_patterns.kernel_lambda
+    PRIVATE libcudacxx::libcudacxx
+  )
+endif()

cccl_upstream/examples/ccclrt/kernel_launch_patterns/README.md

@@ -0,0 +1,3 @@
+# Kernel Launch Patterns
+
+This example showcases how kernels and kernel functors can be launched using the `cuda::launch` function.

cccl_upstream/examples/ccclrt/kernel_launch_patterns/common.cuh

@@ -0,0 +1,66 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef COMMON_CUH
+#define COMMON_CUH
+
+#include <cuda/std/algorithm>
+#include <cuda/std/cstddef>
+#include <cuda/std/string_view>
+
+#include <stdio.h>
+
+class KernelName
+{
+  static constexpr cuda::std::size_t max_size = 128;
+
+  char name_[max_size]; // The name buffer.
+
+public:
+  __host__ __device__ KernelName(cuda::std::string_view name)
+  {
+    assert(name.size() < max_size);
+
+    // Copy the name.
+    cuda::std::copy_n(name.data(), name.size(), name_);
+
+    // Zero terminate the string.
+    name_[name.size()] = '\0';
+  }
+
+  // Returns the stored name.
+  __host__ __device__ const char* get() const
+  {
+    return name_;
+  }
+};
+
+__device__ void say_hello(uint3 from_tindex, const KernelName& kernel_name)
+{
+  const auto this_tindex = cuda::gpu_thread.index(cuda::block);
+
+  printf("[%u, %u]: Hello from thread [%u, %u] launched as %s!\n",
+         this_tindex.x,
+         this_tindex.y,
+         from_tindex.x,
+         from_tindex.y,
+         kernel_name.get());
+
+  // Wait for all threads in block to print the output.
+  __syncthreads();
+
+  // Print additional new line once.
+  if (this_tindex.x == 0 && this_tindex.y == 0)
+  {
+    printf("\n");
+  }
+}
+
+#endif // COMMON_CUH

cccl_upstream/examples/ccclrt/kernel_launch_patterns/kernel.cu

@@ -0,0 +1,148 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+// This example demonstrates how kernels can be launched using cuda::launch.
+
+#include <cuda/devices>
+#include <cuda/hierarchy>
+#include <cuda/launch>
+#include <cuda/stream>
+
+#include <cstdio>
+#include <exception>
+
+#include "common.cuh"
+
+// Regular kernel.
+__global__ void kernel(KernelName kernel_name)
+{
+  // Call say_hello with this thread's index.
+  say_hello(cuda::gpu_thread.index(cuda::block), kernel_name);
+}
+
+// Regular kernel with dynamic shared memory.
+__global__ void kernel_with_dynamic_smem(KernelName kernel_name)
+{
+  // Get the dynamic shared memory handle.
+  extern __shared__ uint3 smem[];
+
+  // Get all necessary hierarchy values.
+  const auto tindex = cuda::gpu_thread.index(cuda::block);
+  const auto trank  = cuda::gpu_thread.rank(cuda::block);
+  const auto tcount = cuda::gpu_thread.count(cuda::block);
+
+  // Each thread will write it's index to the next thread's index in the shared memory.
+  smem[(trank + 1) % tcount] = tindex;
+
+  // Wait for the all threads to finish the write to shared memory.
+  __syncthreads();
+
+  // Call say_hello with previous thread's index.
+  say_hello(smem[trank], kernel_name);
+}
+
+// Kernel that takes cuda::kernel_config as the first parameter. That way the kernel has access to compile time
+// information of the block and grid dimensions which can produce better optimized kernels.
+template <class Config>
+__global__ void kernel_with_config(Config config, KernelName kernel_name)
+{
+  // Call say_hello with this thread's index. Note that passing config to hierarchy queries can improve their
+  // performance since the functionality has access to the compile time specified dimensions.
+  say_hello(cuda::gpu_thread.index(cuda::block, config), kernel_name);
+}
+
+// Kernel that takes cuda::kernel_config that contains the cuda::dynamic_shared_memory_option.
+template <class Config>
+__global__ void kernel_with_config_and_dynamic_smem(Config config, KernelName kernel_name)
+{
+  // Retrieve the dynamic shared memory view. Since we passed uint3[4], we will get cuda::std::span<uint3>.
+  const auto smem = cuda::dynamic_shared_memory(config);
+
+  // Get all necessary hierarchy values.
+  const auto tindex = cuda::gpu_thread.index(cuda::block, config);
+  const auto trank  = cuda::gpu_thread.rank(cuda::block, config);
+  const auto tcount = cuda::gpu_thread.count(cuda::block, config);
+
+  // Each thread will write it's index to the next thread's index in the shared memory.
+  smem[(trank + 1) % tcount] = tindex;
+
+  // Wait for the all threads to finish the write to shared memory.
+  __syncthreads();
+
+  // Call say hello with received previous thread's index.
+  say_hello(smem[trank], kernel_name);
+}
+
+int main()
+try
+{
+  // Check we have at least one device.
+  if (cuda::devices.size() == 0)
+  {
+    std::fprintf(stderr, "No CUDA devices found\n");
+    return 1;
+  }
+
+  // We will use the first device.
+  cuda::device_ref device = cuda::devices[0];
+
+  // cuda::launch always requires a work submitter, so let's create a CUDA stream.
+  cuda::stream stream{device};
+
+  // Set block and grid dimensions to be used with the kernel config. Dimensions specified as template parameters will
+  // be statically known in the kernel.
+  const auto block_dims = cuda::block_dims<2, 2>();
+  const auto grid_dims  = cuda::grid_dims(dim3{1});
+
+  // Make the kernel config.
+  const auto kernel_config = cuda::make_config(grid_dims, block_dims);
+
+  // For kernels that use dynamic shared memory, we need a dynamic shared memory option to be passed in the kernel
+  // config.
+  const auto dyn_smem_opt = cuda::dynamic_shared_memory<uint3[]>(cuda::gpu_thread.count(cuda::block, kernel_config));
+
+  // Make the kernel config with dynamic shared memory option.
+  const auto kernel_config_with_dyn_smem = cuda::make_config(grid_dims, block_dims, dyn_smem_opt);
+
+  // Launch the kernel using the kernel config.
+  cuda::launch(stream, kernel_config, kernel, KernelName{"kernel"});
+
+  // Launch the kernel using the kernel config with dynamic shared memory option.
+  cuda::launch(
+    stream, kernel_config_with_dyn_smem, kernel_with_dynamic_smem, KernelName{"kernel with dynamic shared memory"});
+
+  // Launching kernels with template parameters is more complicated. All of the template parameters must be specified to
+  // obtain the kernel address. Kernel functors can simplify this case a lot.
+  cuda::launch(stream, kernel_config, kernel_with_config<decltype(kernel_config)>, KernelName{"kernel with config"});
+
+  // Kernels with configs that contain dynamic shared memory option must be launched similarly.
+  cuda::launch(stream,
+               kernel_config_with_dyn_smem,
+               kernel_with_config_and_dynamic_smem<decltype(kernel_config_with_dyn_smem)>,
+               KernelName{"kernel with config and dynamic shared memory"});
+
+  // Wait for all of the tasks in the stream to complete.
+  stream.sync();
+}
+catch (const cuda::cuda_error& e)
+{
+  std::fprintf(stderr, "CUDA error: %s\n", e.what());
+  return 1;
+}
+catch (const std::exception& e)
+{
+  std::fprintf(stderr, "Error: %s\n", e.what());
+  return 1;
+}
+catch (...)
+{
+  std::fprintf(stderr, "An unknown error was encountered\n");
+  return 1;
+}

cccl_upstream/examples/ccclrt/kernel_launch_patterns/kernel_functor.cu

@@ -0,0 +1,172 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+// This example demonstrates how kernel functors can be launched using cuda::launch.
+
+#include <cuda/devices>
+#include <cuda/hierarchy>
+#include <cuda/launch>
+#include <cuda/stream>
+
+#include <cstdio>
+#include <exception>
+
+#include "common.cuh"
+
+// This is a kernel functor, a callable object with operator() decorated with __device__ attribute. When launched, the
+// object is copied to the device and operator() is invoked on the device.
+struct KernelFunctor
+{
+  // The operator() must be decorated with __device__ attribute. It can also be a template.
+  __device__ void operator()(const KernelName& kernel_name) const
+  {
+    say_hello(cuda::gpu_thread.index(cuda::block), kernel_name);
+  }
+};
+
+// Kernel functor can contain data. However, the functor must be trivially copyable.
+struct KernelFunctorWithData
+{
+  KernelName kernel_name_;
+
+  __device__ void operator()() const
+  {
+    say_hello(cuda::gpu_thread.index(cuda::block), kernel_name_);
+  }
+};
+
+// Kernel functors can also take the cuda::kernel_config objects as the first argument. That way the kernel has access
+// to compile time information of the block and grid dimensions which can produce better optimized kernels.
+struct KernelFunctorWithConfig
+{
+  template <class Config>
+  __device__ void operator()(const Config& config, const KernelName& kernel_name) const
+  {
+    // The config can be used in hierarchy queries for better performance.
+    say_hello(cuda::gpu_thread.index(cuda::block, config), kernel_name);
+  }
+};
+
+// Kernel functors can provide a default config that is combined with the config passed to cuda::launch. This can be
+// useful for example when a kernel functor requires cooperative launch.
+struct KernelFunctorWithDefaultConfig
+{
+  // Kernel functor provides the default config by implementing the .default_config() method.
+  auto default_config() const
+  {
+    // This default config only specifies that the block dimensions are 2x2. The config passed to cuda::launch must
+    // provide grid dimensions, otherwise the kernel functor wouldn't be able to be launched.
+    return cuda::make_config(cuda::block_dims<2, 2>());
+  }
+
+  template <class Config>
+  __device__ void operator()(const Config& config, const KernelName& kernel_name) const
+  {
+    say_hello(cuda::gpu_thread.index(cuda::block, config), kernel_name);
+  }
+};
+
+// Kernel functor can use the
+struct KernelFunctorWithDynamicSmem
+{
+  template <class Config>
+  __device__ void operator()(const Config& config, const KernelName& kernel_name) const
+  {
+    // Retrieve the dynamic shared memory view. Since we passed uint3[4], we will get cuda::std::span<uint3>.
+    const auto smem = cuda::dynamic_shared_memory(config);
+
+    // Get all necessary hierarchy values.
+    const auto tindex = cuda::gpu_thread.index(cuda::block, config);
+    const auto trank  = cuda::gpu_thread.rank(cuda::block, config);
+    const auto tcount = cuda::gpu_thread.count(cuda::block, config);
+
+    // Each thread will write it's index to the next thread's index in the shared memory.
+    smem[(trank + 1) % tcount] = tindex;
+
+    // Wait for the all threads to finish the write to shared memory.
+    __syncthreads();
+
+    // Call say hello with received previous thread's index.
+    say_hello(smem[trank], kernel_name);
+  }
+};
+
+int main()
+try
+{
+  // Check we have at least one device.
+  if (cuda::devices.size() == 0)
+  {
+    std::fprintf(stderr, "No CUDA devices found\n");
+    return 1;
+  }
+
+  // We will use the first device.
+  cuda::device_ref device = cuda::devices[0];
+
+  // cuda::launch always requires a work submitter, so let's create a CUDA stream.
+  cuda::stream stream{device};
+
+  // Set block and grid dimensions to be used with the kernel config. Dimensions specified as template parameters will
+  // be statically known in the kernel.
+  const auto block_dims = cuda::block_dims<2, 2>();
+  const auto grid_dims  = cuda::grid_dims(dim3{1});
+
+  // Make the kernel config.
+  const auto kernel_config = cuda::make_config(grid_dims, block_dims);
+
+  // For kernels that use dynamic shared memory, we need a dynamic shared memory option to be passed in the kernel
+  // config.
+  const auto dyn_smem_opt = cuda::dynamic_shared_memory<uint3[]>(cuda::gpu_thread.count(cuda::block, kernel_config));
+
+  // Make the kernel config with dynamic shared memory option.
+  const auto kernel_config_with_dyn_smem = cuda::make_config(grid_dims, block_dims, dyn_smem_opt);
+
+  // Launch the kernel functor using the kernel config.
+  cuda::launch(stream, kernel_config, KernelFunctor{}, KernelName{"kernel functor"});
+
+  // Kernel functor can also contain data.
+  cuda::launch(stream, kernel_config, KernelFunctorWithData{KernelName{"kernel functor with data"}});
+
+  // If the kernel functor in invocable with the kernel config, it's automatically passed as the first parameter by the
+  // cuda::launch function.
+  cuda::launch(stream, kernel_config, KernelFunctorWithConfig{}, KernelName{"kernel functor with config"});
+
+  // When launching a kernel functor with default config, we need to pass just a partial config as the launch parameter.
+  // The missing parts are supplied from the default config inside the cuda::launch function.
+  cuda::launch(stream,
+               cuda::make_config(grid_dims),
+               KernelFunctorWithDefaultConfig{},
+               KernelName{"kernel functor with default config"});
+
+  // Launch the kernel functor that uses dynamic shared memory.
+  cuda::launch(stream,
+               kernel_config_with_dyn_smem,
+               KernelFunctorWithDynamicSmem{},
+               KernelName{"kernel functor with dynamic shared memory"});
+
+  // Wait for all of the tasks in the stream to complete.
+  stream.sync();
+}
+catch (const cuda::cuda_error& e)
+{
+  std::fprintf(stderr, "CUDA error: %s\n", e.what());
+  return 1;
+}
+catch (const std::exception& e)
+{
+  std::fprintf(stderr, "Error: %s\n", e.what());
+  return 1;
+}
+catch (...)
+{
+  std::fprintf(stderr, "An unknown error was encountered\n");
+  return 1;
+}

cccl_upstream/examples/ccclrt/kernel_launch_patterns/kernel_lambda.cu

@@ -0,0 +1,122 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of libcu++, the C++ Standard Library for your entire system,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+// This example demonstrates how kernel lambdas can be launched using cuda::launch. Kernel lambdas behave mostly the
+// same way as other kernel functors, but there are some differences.
+
+#if !defined(__CUDACC_EXTENDED_LAMBDA__)
+#  error "This example requires extended lambda support."
+#endif // !__CUDACC_EXTENDED_LAMBDA__
+
+#include <cuda/devices>
+#include <cuda/hierarchy>
+#include <cuda/launch>
+#include <cuda/stream>
+
+#include <cstdio>
+#include <exception>
+
+#include "common.cuh"
+
+int main()
+try
+{
+  // Check we have at least one device.
+  if (cuda::devices.size() == 0)
+  {
+    std::fprintf(stderr, "No CUDA devices found\n");
+    return 1;
+  }
+
+  // We will use the first device.
+  cuda::device_ref device = cuda::devices[0];
+
+  // cuda::launch always requires a work submitter, so let's create a CUDA stream.
+  cuda::stream stream{device};
+
+  // Set block and grid dimensions to be used with the kernel config. Dimensions specified as template parameters will
+  // be statically known in the kernel.
+  const auto block_dims = cuda::block_dims<2, 2>();
+  const auto grid_dims  = cuda::grid_dims(dim3{1});
+
+  // Make the kernel config.
+  const auto kernel_config = cuda::make_config(grid_dims, block_dims);
+
+  // For kernels that use dynamic shared memory, we need a dynamic shared memory option to be passed in the kernel
+  // config.
+  const auto dyn_smem_opt = cuda::dynamic_shared_memory<uint3[]>(cuda::gpu_thread.count(cuda::block, kernel_config));
+
+  // Make the kernel config with dynamic shared memory option.
+  const auto kernel_config_with_dyn_smem = cuda::make_config(grid_dims, block_dims, dyn_smem_opt);
+
+  // Launch the kernel lambda using the kernel config. Unlike with kernel functors, the config is not automatically
+  // passed as the first parameter and must be passed explicitly.
+  cuda::launch(
+    stream,
+    kernel_config,
+    [] __device__(auto config, auto kernel_name) {
+      say_hello(cuda::gpu_thread.index(cuda::block, config), kernel_name);
+    },
+    kernel_config,
+    KernelName{"kernel lambda"});
+
+  // The kernel lambda with captures be launched the same way. All parameters must be captured by value and each thread
+  // will get a copy of the lambda.
+  cuda::launch(
+    stream,
+    kernel_config,
+    [kernel_name = KernelName{"kernel lambda with capture"}] __device__(auto config) {
+      say_hello(cuda::gpu_thread.index(cuda::block, config), kernel_name);
+    },
+    kernel_config);
+
+  // The kernel lambda can use dynamic shared memory in the same way as kernel functors.
+  cuda::launch(
+    stream,
+    kernel_config_with_dyn_smem,
+    [] __device__(auto config, auto kernel_name) {
+      // Retrieve the dynamic shared memory view. Since we passed uint3[4], we will get cuda::std::span<uint3>.
+      const auto smem = cuda::dynamic_shared_memory(config);
+
+      // Get all necessary hierarchy values.
+      const auto tindex = cuda::gpu_thread.index(cuda::block, config);
+      const auto trank  = cuda::gpu_thread.rank(cuda::block, config);
+      const auto tcount = cuda::gpu_thread.count(cuda::block, config);
+
+      // Each thread will write it's index to the next thread's index in the shared memory.
+      smem[(trank + 1) % tcount] = tindex;
+
+      // Wait for the all threads to finish the write to shared memory.
+      __syncthreads();
+
+      // Call say hello with received previous thread's index.
+      say_hello(smem[trank], kernel_name);
+    },
+    kernel_config_with_dyn_smem,
+    KernelName{"kernel lambda with dynamic shared memory"});
+
+  // Wait for all of the tasks in the stream to complete.
+  stream.sync();
+}
+catch (const cuda::cuda_error& e)
+{
+  std::fprintf(stderr, "CUDA error: %s\n", e.what());
+  return 1;
+}
+catch (const std::exception& e)
+{
+  std::fprintf(stderr, "Error: %s\n", e.what());
+  return 1;
+}
+catch (...)
+{
+  std::fprintf(stderr, "An unknown error was encountered\n");
+  return 1;
+}