[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/cudax/test/stf/freeze/constant_logical_data.cu

@@ -0,0 +1,72 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDASTF in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+/**
+ * @file
+ *
+ * @brief Helper to build constant data based on frozen logical data
+ *
+ */
+
+#include <cuda/experimental/__stf/utility/constant_logical_data.cuh>
+#include <cuda/experimental/stf.cuh>
+
+using namespace cuda::experimental::stf;
+
+int main()
+{
+  stream_ctx ctx;
+
+  const int N = 16;
+
+  /* Create a constant value */
+  auto ld_cst = ctx.logical_data(shape_of<slice<int>>(N));
+  ctx.parallel_for(ld_cst.shape(), ld_cst.write())->*[] __device__(size_t i, slice<int> res) {
+    res(i) = 18 * i - 9;
+  };
+  auto cst = constant_logical_data(ctx, mv(ld_cst));
+
+  int X[N];
+
+  for (int i = 0; i < N; i++)
+  {
+    X[i] = 5 * i - 3;
+  }
+
+  auto lX = ctx.logical_data(X).set_symbol("X");
+
+  for (size_t iter = 0; iter < 4; iter++)
+  {
+    auto cst_slice = cst.get();
+    ctx.parallel_for(lX.shape(), lX.rw()).set_symbol("X+=cst")->*[cst_slice] __device__(size_t i, auto x) {
+      x(i) += cst_slice(i);
+    };
+
+    auto cst2 = run_once()->*[&]() {
+      auto ld = ctx.logical_data(shape_of<slice<int>>(N));
+      ctx.parallel_for(ld.shape(), ld.write())->*[] __device__(size_t i, slice<int> res) {
+        res(i) = 4 * i - 2;
+      };
+      return constant_logical_data(ctx, mv(ld));
+    };
+
+    auto cst2_slice = cst2.get();
+    ctx.parallel_for(lX.shape(), lX.rw()).set_symbol("X+=cst2")->*[cst2_slice] __device__(size_t i, auto x) {
+      x(i) += cst2_slice(i);
+    };
+  }
+
+  ctx.finalize();
+
+  for (int i = 0; i < N; i++)
+  {
+    EXPECT(X[i] == (5 * i - 3) + 4 * (18 * i - 9) + 4 * (4 * i - 2));
+  }
+}

cccl_upstream/cudax/test/stf/freeze/freeze.cu

@@ -0,0 +1,82 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDASTF in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+/**
+ * @file
+ *
+ * @brief Freeze data in read-only fashion
+ *
+ */
+
+#include <cuda/experimental/__stf/graph/graph_ctx.cuh>
+#include <cuda/experimental/__stf/stream/stream_ctx.cuh>
+
+using namespace cuda::experimental::stf;
+
+int X0(int i)
+{
+  return 17 * i + 45;
+}
+
+__global__ void print(slice<int> s)
+{
+  int tid      = blockIdx.x * blockDim.x + threadIdx.x;
+  int nthreads = gridDim.x * blockDim.x;
+
+  for (int i = tid; i < s.size(); i += nthreads)
+  {
+    printf("%d %d\n", i, s(i));
+  }
+}
+
+int main()
+{
+  stream_ctx ctx;
+
+  cudaStream_t stream = ctx.pick_stream();
+
+  const int N = 16;
+  int X[N];
+
+  for (int i = 0; i < N; i++)
+  {
+    X[i] = X0(i);
+  }
+
+  auto lX = ctx.logical_data(X).set_symbol("X");
+  auto lY = ctx.logical_data(lX.shape()).set_symbol("Y");
+
+  ctx.parallel_for(lX.shape(), lX.rw()).set_symbol("X=2X")->*[] __device__(size_t i, auto x) {
+    x(i) *= 2;
+  };
+
+  auto fx = ctx.freeze(lX);
+
+  auto dX = fx.get(data_place::current_device(), stream);
+
+  print<<<8, 4, 0, stream>>>(dX);
+
+  ctx.parallel_for(lX.shape(), lX.read(), lY.write()).set_symbol("Y=X")->*[] __device__(size_t i, auto x, auto y) {
+    y(i) = x(i);
+  };
+
+  fx.unfreeze(stream);
+
+  ctx.parallel_for(lX.shape(), lX.rw()).set_symbol("X+=1")->*[] __device__(size_t i, auto x) {
+    x(i) += 1;
+  };
+
+  ctx.finalize();
+
+  for (int i = 0; i < N; i++)
+  {
+    EXPECT(X[i] == 2 * X0(i) + 1);
+  }
+}

cccl_upstream/cudax/test/stf/freeze/freeze_rw.cu

@@ -0,0 +1,86 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDASTF in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+/**
+ * @file
+ *
+ * @brief Freeze data in read-only fashion
+ *
+ */
+
+#include <cuda/experimental/__stf/graph/graph_ctx.cuh>
+#include <cuda/experimental/__stf/stream/stream_ctx.cuh>
+
+using namespace cuda::experimental::stf;
+
+int X0(int i)
+{
+  return 17 * i + 45;
+}
+
+__global__ void mult(slice<int> s, int val)
+{
+  int tid      = blockIdx.x * blockDim.x + threadIdx.x;
+  int nthreads = gridDim.x * blockDim.x;
+
+  for (int i = tid; i < s.size(); i += nthreads)
+  {
+    s(i) *= val;
+  }
+}
+
+int main()
+{
+  stream_ctx ctx;
+
+  cudaStream_t stream = ctx.pick_stream();
+
+  const int N = 16;
+  int X[N];
+
+  for (int i = 0; i < N; i++)
+  {
+    X[i] = X0(i);
+  }
+
+  auto lX = ctx.logical_data(X).set_symbol("X");
+  auto lY = ctx.logical_data(lX.shape()).set_symbol("Y");
+
+  for (int k = 0; k < 4; k++)
+  {
+    auto fx = ctx.freeze(lX, access_mode::rw, data_place::current_device());
+
+    _CCCL_ASSERT(fx.get_access_mode() == access_mode::rw, "invalid access mode");
+
+    auto dX = fx.get(data_place::current_device(), stream);
+    mult<<<8, 4, 0, stream>>>(dX, 4);
+    fx.unfreeze(stream);
+
+    ctx.parallel_for(lX.shape(), lX.read(), lY.write()).set_symbol("Y=X")->*[] __device__(size_t i, auto x, auto y) {
+      y(i) = x(i);
+    };
+
+    ctx.parallel_for(lX.shape(), lY.rw()).set_symbol("Y+=1")->*[] __device__(size_t i, auto y) {
+      y(i) += 1;
+    };
+
+    //        ctx.host_launch(lX.read(), lY.read())->*[](auto x, auto y) {
+    //            for (int i = 0; i < x.size(); i++) {
+    //                EXPECT(x(i) == 2*X0(i) + 4);
+    //            }
+    //
+    //            for (int i = 0; i < y.size(); i++) {
+    //                EXPECT(y(i) == 2*X0(i) + 4);
+    //            }
+    //        };
+  }
+
+  ctx.finalize();
+}

cccl_upstream/cudax/test/stf/freeze/freeze_untyped_rw.cu

@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDASTF in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+//! \file
+//!
+//! \brief Freeze data and store it as a frozen_logical_data_untyped object
+
+#include <cuda/experimental/__stf/graph/graph_ctx.cuh>
+#include <cuda/experimental/__stf/stream/stream_ctx.cuh>
+
+using namespace cuda::experimental::stf;
+
+int X0(int i)
+{
+  return 17 * i + 45;
+}
+
+__global__ void mult(slice<int> s, int val)
+{
+  int tid      = blockIdx.x * blockDim.x + threadIdx.x;
+  int nthreads = gridDim.x * blockDim.x;
+
+  for (int i = tid; i < s.size(); i += nthreads)
+  {
+    s(i) *= val;
+  }
+}
+
+int main()
+{
+  stream_ctx ctx;
+
+  cudaStream_t stream = ctx.pick_stream();
+
+  const int N = 16;
+  int X[N];
+
+  for (int i = 0; i < N; i++)
+  {
+    X[i] = X0(i);
+  }
+
+  auto lX = ctx.logical_data(X).set_symbol("X");
+  auto lY = ctx.logical_data(lX.shape()).set_symbol("Y");
+
+  for (int k = 0; k < 4; k++)
+  {
+    logical_data_untyped lX_untyped = lX;
+    auto fx                         = ctx.freeze(lX_untyped, access_mode::rw, data_place::current_device());
+
+    _CCCL_ASSERT(fx.get_access_mode() == access_mode::rw, "invalid access mode");
+
+    auto dX = fx.template get<slice<int>>(data_place::current_device(), stream);
+    mult<<<8, 4, 0, stream>>>(dX, 4);
+    fx.unfreeze(stream);
+
+    ctx.parallel_for(lX.shape(), lX.read(), lY.write()).set_symbol("Y=X")->*[] __device__(size_t i, auto x, auto y) {
+      y(i) = x(i);
+    };
+
+    ctx.parallel_for(lX.shape(), lY.rw()).set_symbol("Y+=1")->*[] __device__(size_t i, auto y) {
+      y(i) += 1;
+    };
+  }
+
+  ctx.finalize();
+}

cccl_upstream/cudax/test/stf/freeze/freeze_write_back.cu

@@ -0,0 +1,65 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDASTF in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+//! \file
+//!
+//! \brief Ensure write-back is working on logical data alias made by freezing another one
+
+#include <cuda/experimental/stf.cuh>
+
+using namespace cuda::experimental::stf;
+
+int main()
+{
+  context ctx;
+
+  int array[1024];
+  for (size_t i = 0; i < 1024; i++)
+  {
+    array[i] = 2 - i * i;
+  }
+
+  auto lA = ctx.logical_data(array).set_symbol("A");
+
+  auto stream = ctx.pick_stream();
+
+  graph_ctx gctx(stream);
+
+  // Create an alias for lA in the graph by freezing it and creating a new
+  // local logical data in the graph.
+  auto fa   = ctx.freeze(lA, access_mode::rw, data_place::current_device());
+  auto inst = fa.get(data_place::current_device(), stream);
+  auto glA  = gctx.logical_data(inst, data_place::current_device());
+
+  gctx.parallel_for(glA.shape(), glA.rw())->*[] __device__(size_t i, auto a) {
+    a(i) += 4 * i;
+  };
+
+  // force to move to a different place, and probably to allocate another copy
+  // on the host. This tests if the write-back mechanism works from the host to
+  // the device when destroying the alias logical data glA.
+  gctx.host_launch(glA.rw())->*[](auto a) {
+    for (size_t i = 0; i < 1024; i++)
+    {
+      a(i) *= 2;
+    }
+  };
+
+  gctx.finalize();
+
+  fa.unfreeze(stream);
+
+  ctx.finalize();
+
+  for (size_t i = 0; i < 1024; i++)
+  {
+    EXPECT(array[i] == 2 * (2 - i * i + 4 * i));
+  }
+}

cccl_upstream/cudax/test/stf/freeze/task_fence.cu

@@ -0,0 +1,98 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDASTF in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+/**
+ * @file
+ *
+ * @brief Freeze data in read-only fashion
+ *
+ */
+
+#include <cuda/experimental/__stf/graph/graph_ctx.cuh>
+#include <cuda/experimental/__stf/stream/stream_ctx.cuh>
+
+using namespace cuda::experimental::stf;
+
+int X0(int i)
+{
+  return 17 * i + 45;
+}
+
+__global__ void print(slice<int> s)
+{
+  int tid      = blockIdx.x * blockDim.x + threadIdx.x;
+  int nthreads = gridDim.x * blockDim.x;
+
+  for (int i = tid; i < s.size(); i += nthreads)
+  {
+    printf("%d %d\n", i, s(i));
+  }
+}
+
+int main()
+{
+  stream_ctx ctx;
+
+  const int N = 16;
+  int X[N];
+
+  for (int i = 0; i < N; i++)
+  {
+    X[i] = X0(i);
+  }
+
+  auto lX = ctx.logical_data(X).set_symbol("X");
+  auto lY = ctx.logical_data(lX.shape()).set_symbol("Y");
+
+  ctx.parallel_for(lX.shape(), lX.rw()).set_symbol("X=2X")->*[] __device__(size_t i, auto x) {
+    x(i) *= 2;
+  };
+
+  // test 1 : implicit sync of gets
+  {
+    auto fx      = ctx.freeze(lX);
+    auto [dX, _] = fx.get(data_place::current_device());
+
+    // the stream returned by fence should depend on the get operation
+    auto stream2 = ctx.fence();
+    print<<<8, 4, 0, stream2>>>(dX);
+
+    ctx.parallel_for(lX.shape(), lX.read(), lY.write()).set_symbol("Y=X")->*[] __device__(size_t i, auto x, auto y) {
+      y(i) = x(i);
+    };
+
+    fx.unfreeze(stream2);
+  }
+
+  // test 2 : unfreeze with no events due to user sync
+  {
+    auto fx      = ctx.freeze(lX);
+    auto [dX, _] = fx.get(data_place::current_device());
+
+    // the stream returned by fence should depend on the get operation
+    auto stream2 = ctx.fence();
+    print<<<8, 4, 0, stream2>>>(dX);
+
+    // We synchronize so there is nothing to depend on anymore
+    cudaStreamSynchronize(stream2);
+    fx.unfreeze(event_list());
+  }
+
+  ctx.parallel_for(lX.shape(), lX.rw()).set_symbol("X+=1")->*[] __device__(size_t i, auto x) {
+    x(i) += 1;
+  };
+
+  ctx.finalize();
+
+  for (int i = 0; i < N; i++)
+  {
+    EXPECT(X[i] == 2 * X0(i) + 1);
+  }
+}

cccl_upstream/cudax/test/stf/freeze/token.cu

@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of CUDASTF in CUDA C++ Core Libraries,
+// under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+// SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES.
+//
+//===----------------------------------------------------------------------===//
+
+/**
+ * @file
+ *
+ * @brief Freeze token
+ *
+ */
+
+#include <cuda/experimental/stf.cuh>
+
+using namespace cuda::experimental::stf;
+
+int main()
+{
+  context ctx;
+
+  auto ltoken = ctx.token();
+
+  auto ftoken = ctx.freeze(ltoken);
+
+  cudaStream_t stream = ctx.pick_stream();
+  // This makes any future operations in this CUDA stream depend on the
+  // availability of the token
+  [[maybe_unused]] auto dtoken = ftoken.get(data_place::current_device(), stream);
+  ftoken.unfreeze(stream);
+
+  ctx.finalize();
+}