[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/c/parallel/src/kernels/iterators.cpp

@@ -0,0 +1,208 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <format>
+
+#include "cccl/c/types.h"
+#include <kernels/iterators.h>
+#include <util/errors.h>
+#include <util/types.h>
+
+constexpr std::string_view format_template = R"XXX(
+#define DIFF_T {0}
+#define OP_ALIGNMENT {1}
+#define OP_SIZE {2}
+#define VALUE_T {3}
+#define DEREF {4}
+#define ADVANCE {5}
+
+// Kernel Source
+{6}
+
+#undef DIFF_T
+#undef OP_ALIGNMENT
+#undef OP_SIZE
+#undef VALUE_T
+#undef DEREF
+#undef ADVANCE
+)XXX";
+
+std::string make_kernel_input_iterator(
+  std::string_view diff_t,
+  size_t alignment,
+  size_t size,
+  std::string_view iterator_name,
+  std::string_view value_t,
+  std::string_view deref,
+  std::string_view advance)
+{
+  const std::string iter_def = std::format(R"XXX(
+extern "C" __device__ void DEREF(const void *self_ptr, void* result);
+extern "C" __device__ void ADVANCE(void *self_ptr, const void* offset);
+struct __align__(OP_ALIGNMENT) {0} {{
+  using iterator_category = cuda::std::random_access_iterator_tag;
+  using value_type = VALUE_T;
+  using difference_type = DIFF_T;
+  using pointer = VALUE_T*;
+  using reference = VALUE_T&;
+  __device__ inline value_type operator*() const {{
+    value_type result;
+    DEREF(data, &result);
+    return result;
+  }}
+  __device__ inline {0}& operator+=(difference_type diff) {{
+      ADVANCE(data, &diff);
+      return *this;
+  }}
+  __device__ inline value_type operator[](difference_type diff) const {{
+      return *(*this + diff);
+  }}
+  __device__ inline {0} operator+(difference_type diff) const {{
+      {0} result = *this;
+      result += diff;
+      return result;
+  }}
+  char data[OP_SIZE];
+}};
+)XXX",
+                                           iterator_name);
+
+  return std::format(format_template, diff_t, alignment, size, value_t, deref, advance, iter_def);
+};
+
+std::string make_kernel_input_iterator(
+  std::string_view offset_t, std::string_view iterator_name, std::string_view input_value_t, cccl_iterator_t iter)
+{
+  if (iter.type == cccl_iterator_kind_t::CCCL_POINTER)
+  {
+    return {};
+  }
+
+  return make_kernel_input_iterator(
+    offset_t, iter.alignment, iter.size, iterator_name, input_value_t, iter.dereference.name, iter.advance.name);
+}
+
+std::string make_kernel_output_iterator(
+  std::string_view diff_t,
+  size_t alignment,
+  size_t size,
+  std::string_view iterator_name,
+  std::string_view value_t,
+  std::string_view deref,
+  std::string_view advance)
+{
+  const std::string iter_def = std::format(R"XXX(
+extern "C" __device__ void DEREF(void *self_ptr, const void* x);
+extern "C" __device__ void ADVANCE(void *self_ptr, const void* offset);
+struct __align__(OP_ALIGNMENT) {0}_state_t {{
+  char data[OP_SIZE];
+}};
+struct {0}_proxy_t {{
+  __device__ {0}_proxy_t operator=(VALUE_T x) {{
+    DEREF(&state, &x);
+    return *this;
+  }}
+  {0}_state_t state;
+}};
+struct {0} {{
+  using iterator_category = cuda::std::random_access_iterator_tag;
+  using difference_type   = DIFF_T;
+  using value_type        = VALUE_T;
+  using pointer           = {0}_proxy_t*;
+  using reference         = {0}_proxy_t;
+  __device__ {0}_proxy_t operator*() const {{ return {{state}}; }}
+  __device__ {0}& operator+=(difference_type diff) {{
+      ADVANCE(&state, &diff);
+      return *this;
+  }}
+  __device__ {0}_proxy_t operator[](difference_type diff) const {{
+    {0} result = *this;
+    result += diff;
+    return {{ result.state }};
+  }}
+  __device__ {0} operator+(difference_type diff) const {{
+    {0} result = *this;
+    result += diff;
+    return result;
+  }}
+  {0}_state_t state;
+}};
+)XXX",
+                                           iterator_name);
+
+  return std::format(format_template, diff_t, alignment, size, value_t, deref, advance, iter_def);
+};
+
+std::string make_kernel_output_iterator(
+  std::string_view offset_t, std::string_view iterator_name, std::string_view input_value_t, cccl_iterator_t iter)
+{
+  if (iter.type == cccl_iterator_kind_t::CCCL_POINTER)
+  {
+    return {};
+  }
+
+  return make_kernel_output_iterator(
+    offset_t, iter.alignment, iter.size, iterator_name, input_value_t, iter.dereference.name, iter.advance.name);
+}
+
+std::string make_kernel_inout_iterator(
+  std::string_view diff_t,
+  size_t alignment,
+  size_t size,
+  std::string_view value_t,
+  std::string_view deref,
+  std::string_view advance)
+{
+  constexpr std::string_view in_out_template = R"XXX(
+extern "C" __device__ {1}* {2}(const void *self_ptr);
+extern "C" __device__ void {3}(void *self_ptr, {0} offset);
+
+struct __align__({5}) output_iterator_state_t{{
+  char data[{4}];
+}};
+
+struct output_iterator_t {{
+  using iterator_category = cuda::std::random_access_iterator_tag;
+  using difference_type   = {0};
+  using value_type        = VALUE_T;
+  using pointer           = output_iterator_proxy_t*;
+  using reference         = output_iterator_proxy_t;
+  __device__ {1} operator*() const {{ return {2}(&state); }}
+  __device__ output_iterator_t& operator+=(difference_type diff) {{
+      {3}(&state, diff);
+      return *this;
+  }}
+  __device__ output_iterator_proxy_t operator[](difference_type diff) const {{
+    output_iterator_t result = *this;
+    result += diff;
+    return {{ result.state }};
+  }}
+  __device__ output_iterator_t operator+(difference_type diff) const {{
+    output_iterator_t result = *this;
+    result += diff;
+    return result;
+  }}
+  output_iterator_state_t state;
+}};
+)XXX";
+
+  return std::format(in_out_template, diff_t, alignment, size, value_t, deref, advance);
+};
+
+std::string make_kernel_inout_iterator(std::string_view offset_t, std::string_view input_value_t, cccl_iterator_t iter)
+{
+  if (iter.type == cccl_iterator_kind_t::CCCL_POINTER)
+  {
+    return {};
+  }
+
+  return make_kernel_inout_iterator(
+    offset_t, iter.alignment, iter.size, input_value_t, iter.dereference.name, iter.advance.name);
+}

cccl_upstream/c/parallel/src/kernels/iterators.h

@@ -0,0 +1,24 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <string>
+#include <string_view>
+
+#include <cccl/c/types.h>
+
+std::string make_kernel_input_iterator(
+  std::string_view offset_t, std::string_view iterator_name, std::string_view input_value_t, cccl_iterator_t iter);
+
+std::string make_kernel_output_iterator(
+  std::string_view offset_t, std::string_view iterator_name, std::string_view input_value_t, cccl_iterator_t iter);
+
+std::string make_kernel_inout_iterator(std::string_view offset_t, std::string_view input_value_t, cccl_iterator_t iter);

cccl_upstream/c/parallel/src/kernels/operators.cpp

@@ -0,0 +1,209 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#include <format>
+#include <string_view>
+#include <unordered_set>
+
+#include "cccl/c/types.h"
+#include <jit_templates/templates/operation.h>
+#include <kernels/operators.h>
+#include <util/errors.h>
+#include <util/types.h>
+
+std::unordered_set<std::string_view> primitive_types = {
+  "::cuda::std::int8_t",
+  "::cuda::std::uint8_t",
+  "::cuda::std::int16_t",
+  "::cuda::std::uint16_t",
+  "::cuda::std::int32_t",
+  "::cuda::std::uint32_t",
+  "::cuda::std::int64_t",
+  "::cuda::std::uint64_t",
+  "__half",
+  "float",
+  "double",
+  "bool",
+};
+
+constexpr std::string_view binary_op_template = R"XXX(
+#define LHS_T {0}
+#define RHS_T {1}
+#define OP_NAME {2}
+#define OP_ALIGNMENT {3}
+#define OP_SIZE {4}
+
+// Source
+{5}
+
+#undef LHS_T
+#undef RHS_T
+#undef OP_NAME
+#undef OP_ALIGNMENT
+#undef OP_SIZE
+)XXX";
+
+constexpr std::string_view stateless_binary_op_template = R"XXX(
+extern "C" __device__ void OP_NAME(const void* lhs, const void* rhs, void* out);
+struct op_wrapper {{
+  __device__ {0} operator()(const LHS_T& lhs, const RHS_T& rhs) const {{
+    {0} ret;
+    OP_NAME(&lhs, &rhs, &ret);
+    return ret;
+  }}
+}};
+)XXX";
+
+constexpr std::string_view stateful_binary_op_template = R"XXX(
+struct __align__(OP_ALIGNMENT) op_state {{
+  char data[OP_SIZE];
+}};
+extern "C" __device__ void OP_NAME(void* state, const void* lhs, const void* rhs, void* out);
+struct op_wrapper {{
+  op_state state;
+  __device__ {0} operator()(const LHS_T& lhs, const RHS_T& rhs) {{
+    {0} ret;
+    OP_NAME(&state, &lhs, &rhs, &ret);
+    return ret;
+  }}
+}};
+)XXX";
+
+std::string make_kernel_binary_operator_full_source(
+  std::string_view lhs_t, std::string_view rhs_t, cccl_op_t operation, std::string_view return_type)
+{
+  if (lhs_t == rhs_t && (return_type == lhs_t || return_type == "bool"))
+  {
+    auto desc = user_operation_traits::well_known_operation_description(operation.type);
+    if (desc && (desc->symbol || primitive_types.contains(lhs_t)))
+    {
+      if (desc->check != user_operation_traits::binary_predicate_matcher
+          && desc->check != user_operation_traits::binary_matcher)
+      {
+        throw std::runtime_error(
+          std::format("c.parallel: invalid well-known operation '{}' specified for a binary cccl_op_t.",
+                      std::format(desc->name, +"")));
+      }
+
+      std::string ret =
+        std::format("#include <cuda/std/functional>\nusing op_wrapper = {};\n", std::format(desc->name, lhs_t.data()));
+      if (!primitive_types.contains(lhs_t))
+      {
+        std::string_view type_names[] = {return_type, lhs_t, rhs_t};
+        ret += user_operation_traits::binary_builder(cuda::std::span(type_names), *desc->symbol, operation.name);
+      }
+      return ret;
+    }
+  }
+
+  const std::string op_alignment =
+    operation.type == cccl_op_kind_t::CCCL_STATEFUL ? std::format("{}", operation.alignment) : "";
+  const std::string op_size = operation.type == cccl_op_kind_t::CCCL_STATEFUL ? std::format("{}", operation.size) : "";
+
+  return std::format(
+    binary_op_template,
+    lhs_t,
+    rhs_t,
+    operation.name,
+    op_alignment,
+    op_size,
+    operation.type == cccl_op_kind_t::CCCL_STATEFUL
+      ? std::format(stateful_binary_op_template, return_type)
+      : std::format(stateless_binary_op_template, return_type));
+}
+
+std::string make_kernel_user_binary_operator(
+  std::string_view lhs_t, std::string_view rhs_t, std::string_view output_t, cccl_op_t operation)
+{
+  return make_kernel_binary_operator_full_source(lhs_t, rhs_t, operation, output_t);
+}
+
+std::string make_kernel_user_comparison_operator(std::string_view input_t, cccl_op_t operation)
+{
+  return make_kernel_binary_operator_full_source(input_t, input_t, operation, "bool");
+}
+
+std::string make_kernel_user_unary_operator(std::string_view input_t, std::string_view output_t, cccl_op_t operation)
+{
+  constexpr std::string_view unary_op_template = R"XXX(
+#define INPUT_T {0}
+#define OUTPUT_T {1}
+#define OP_NAME {2}
+#define OP_ALIGNMENT {3}
+#define OP_SIZE {4}
+
+// Source
+{5}
+
+#undef INPUT_T
+#undef OUTPUT_T
+#undef OP_NAME
+#undef OP_ALIGNMENT
+#undef OP_SIZE
+)XXX";
+
+  constexpr std::string_view stateless_op = R"XXX(
+extern "C" __device__  void OP_NAME(const void* val, void* result);
+struct op_wrapper {
+  __device__ OUTPUT_T operator()(const INPUT_T& val) const {
+    OUTPUT_T out;
+    OP_NAME(&val, &out);
+    return out;
+  }
+};
+)XXX";
+
+  constexpr std::string_view stateful_op = R"XXX(
+struct __align__(OP_ALIGNMENT) op_state {
+  char data[OP_SIZE];
+};
+extern "C" __device__ void OP_NAME(op_state* state, const void* val, void* result);
+struct op_wrapper
+{
+  op_state state;
+  __device__ OUTPUT_T operator()(const INPUT_T& val)
+  {
+    OUTPUT_T out;
+    OP_NAME(&state, &val, &out);
+    return out;
+  }
+};
+
+)XXX";
+
+  if (output_t == input_t || output_t == "bool")
+  {
+    auto desc = user_operation_traits::well_known_operation_description(operation.type);
+    if (desc && (desc->symbol || primitive_types.contains(input_t)))
+    {
+      if (desc->check != user_operation_traits::unary_predicate_matcher
+          && desc->check != user_operation_traits::unary_matcher)
+      {
+        throw std::runtime_error(
+          std::format("c.parallel: invalid well-known operation '{}' specified for a unary cccl_op_t.",
+                      std::format(desc->name, +"")));
+      }
+
+      std::string_view type_names[] = {output_t, input_t};
+      std::string ret               = std::format(
+        "#include <cuda/std/functional>\nusing op_wrapper = {};\n", std::format(desc->name, input_t.data()));
+      if (!primitive_types.contains(input_t))
+      {
+        ret += user_operation_traits::unary_builder(cuda::std::span(type_names), *desc->symbol, operation.name);
+      }
+      return ret;
+    }
+  }
+
+  return (operation.type == cccl_op_kind_t::CCCL_STATEFUL)
+         ? std::format(
+             unary_op_template, input_t, output_t, operation.name, operation.alignment, operation.size, stateful_op)
+         : std::format(unary_op_template, input_t, output_t, operation.name, "", "", stateless_op);
+}

cccl_upstream/c/parallel/src/kernels/operators.h

@@ -0,0 +1,23 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDA Experimental in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+#pragma once
+
+#include <string>
+
+#include <cccl/c/types.h>
+
+std::string make_kernel_user_binary_operator(
+  std::string_view lhs_value_t, std::string_view rhs_value_t, std::string_view output_value_t, cccl_op_t operation);
+
+std::string
+make_kernel_user_unary_operator(std::string_view input_value_t, std::string_view output_value_t, cccl_op_t operation);
+
+std::string make_kernel_user_comparison_operator(std::string_view input_value_t, cccl_op_t operation);