[INFRA] Import NVIDIA/CCCL upstream as optimization reference library

CCCL (CUDA C++ Core Libraries) provides:
- CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk)
- Thrust: high-level parallel algorithms (transform_reduce, sort, scan)
- libcudacxx: CUDA C++ standard library (atomics, barriers, memory)
- cudax: experimental features (memory resources, allocators)
- Tuning policies: per-SM hardware-specific algorithm parameters

Competition optimization vectors mapped to CCCL:
- Output TPS (83% weight): warp_reduce, block_reduce, device_topk
- Input TPS (14% weight): device_scan, block_load, prefetch
- Cache TPS (3% weight): prefix caching strategy patterns
- Memory (0.9 util): pooled/cached/buddy allocators

Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only)
License: Apache-2.0

cccl_upstream/cub/benchmarks/bench/transform/applications/P1/bfloat16.h

@@ -0,0 +1,235 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#pragma once
+
+#include <cuda_bf16.h>
+
+#include <cmath>
+#include <cstdint>
+#include <cstring>
+
+#include <nvbench/type_strings.cuh>
+
+// ============================================================================
+// BFloat16 type — replicated from c10::BFloat16
+// (torch/headeronly/util/BFloat16.h)
+// ============================================================================
+
+namespace bf16_detail
+{
+inline __host__ __device__ float f32_from_bits(uint16_t src)
+{
+  float res    = 0;
+  uint32_t tmp = src;
+  tmp <<= 16;
+  std::memcpy(&res, &tmp, sizeof(tmp));
+  return res;
+}
+
+inline __host__ __device__ uint16_t round_to_nearest_even(float src)
+{
+  if (std::isnan(src))
+  {
+    return UINT16_C(0x7FC0);
+  }
+  else
+  {
+    uint32_t U32;
+    std::memcpy(&U32, &src, sizeof(U32));
+    uint32_t rounding_bias = ((U32 >> 16) & 1) + UINT32_C(0x7FFF);
+    return static_cast<uint16_t>((U32 + rounding_bias) >> 16);
+  }
+}
+} // namespace bf16_detail
+
+struct alignas(2) BFloat16
+{
+  uint16_t x;
+
+  BFloat16() = default;
+
+  struct from_bits_t
+  {};
+  static constexpr __host__ __device__ from_bits_t from_bits()
+  {
+    return from_bits_t();
+  }
+
+  constexpr __host__ __device__ BFloat16(unsigned short bits, from_bits_t)
+      : x(bits)
+  {}
+
+  /* implicit */ inline __host__ __device__ BFloat16(float value);
+  inline __host__ __device__ operator float() const;
+
+  inline __host__ __device__ BFloat16(const __nv_bfloat16& value);
+  explicit inline __host__ __device__ operator __nv_bfloat16() const;
+};
+
+inline __host__ __device__ BFloat16::BFloat16(float value)
+{
+  NV_IF_ELSE_TARGET(NV_PROVIDES_SM_80,
+                    ({
+                      __nv_bfloat16 tmp = __float2bfloat16(value);
+                      x                 = *reinterpret_cast<const unsigned short*>(&tmp);
+                    }),
+                    ({ x = bf16_detail::round_to_nearest_even(value); }));
+}
+
+inline __host__ __device__ BFloat16::operator float() const
+{
+  return __bfloat162float(*reinterpret_cast<const __nv_bfloat16*>(&x));
+}
+
+inline __host__ __device__ BFloat16::BFloat16(const __nv_bfloat16& value)
+{
+  x = *reinterpret_cast<const unsigned short*>(&value);
+}
+inline __host__ __device__ BFloat16::operator __nv_bfloat16() const
+{
+  return *reinterpret_cast<const __nv_bfloat16*>(&x);
+}
+
+// Arithmetic — BFloat16 x BFloat16 → BFloat16
+inline __host__ __device__ BFloat16 operator+(const BFloat16& a, const BFloat16& b)
+{
+  return static_cast<float>(a) + static_cast<float>(b);
+}
+inline __host__ __device__ BFloat16 operator-(const BFloat16& a, const BFloat16& b)
+{
+  return static_cast<float>(a) - static_cast<float>(b);
+}
+inline __host__ __device__ BFloat16 operator*(const BFloat16& a, const BFloat16& b)
+{
+  return static_cast<float>(a) * static_cast<float>(b);
+}
+inline __host__ __device__ BFloat16 operator/(const BFloat16& a, const BFloat16& b)
+{
+  return static_cast<float>(a) / static_cast<float>(b);
+}
+inline __host__ __device__ BFloat16 operator-(const BFloat16& a)
+{
+  return -static_cast<float>(a);
+}
+
+// Compound assignment — BFloat16
+inline __host__ __device__ BFloat16& operator+=(BFloat16& a, const BFloat16& b)
+{
+  a = a + b;
+  return a;
+}
+inline __host__ __device__ BFloat16& operator-=(BFloat16& a, const BFloat16& b)
+{
+  a = a - b;
+  return a;
+}
+inline __host__ __device__ BFloat16& operator*=(BFloat16& a, const BFloat16& b)
+{
+  a = a * b;
+  return a;
+}
+inline __host__ __device__ BFloat16& operator/=(BFloat16& a, const BFloat16& b)
+{
+  a = a / b;
+  return a;
+}
+
+// Arithmetic — BFloat16 x float → float
+inline __host__ __device__ float operator+(BFloat16 a, float b)
+{
+  return static_cast<float>(a) + b;
+}
+inline __host__ __device__ float operator-(BFloat16 a, float b)
+{
+  return static_cast<float>(a) - b;
+}
+inline __host__ __device__ float operator*(BFloat16 a, float b)
+{
+  return static_cast<float>(a) * b;
+}
+inline __host__ __device__ float operator/(BFloat16 a, float b)
+{
+  return static_cast<float>(a) / b;
+}
+inline __host__ __device__ float operator+(float a, BFloat16 b)
+{
+  return a + static_cast<float>(b);
+}
+inline __host__ __device__ float operator-(float a, BFloat16 b)
+{
+  return a - static_cast<float>(b);
+}
+inline __host__ __device__ float operator*(float a, BFloat16 b)
+{
+  return a * static_cast<float>(b);
+}
+inline __host__ __device__ float operator/(float a, BFloat16 b)
+{
+  return a / static_cast<float>(b);
+}
+
+// Compound assignment — float x BFloat16 → float
+inline __host__ __device__ float& operator+=(float& a, const BFloat16& b)
+{
+  return a += static_cast<float>(b);
+}
+inline __host__ __device__ float& operator-=(float& a, const BFloat16& b)
+{
+  return a -= static_cast<float>(b);
+}
+inline __host__ __device__ float& operator*=(float& a, const BFloat16& b)
+{
+  return a *= static_cast<float>(b);
+}
+inline __host__ __device__ float& operator/=(float& a, const BFloat16& b)
+{
+  return a /= static_cast<float>(b);
+}
+
+// Arithmetic — BFloat16 x int → BFloat16
+inline __host__ __device__ BFloat16 operator+(BFloat16 a, int b)
+{
+  return a + static_cast<BFloat16>(static_cast<float>(b));
+}
+inline __host__ __device__ BFloat16 operator-(BFloat16 a, int b)
+{
+  return a - static_cast<BFloat16>(static_cast<float>(b));
+}
+inline __host__ __device__ BFloat16 operator*(BFloat16 a, int b)
+{
+  return a * static_cast<BFloat16>(static_cast<float>(b));
+}
+inline __host__ __device__ BFloat16 operator/(BFloat16 a, int b)
+{
+  return a / static_cast<BFloat16>(static_cast<float>(b));
+}
+inline __host__ __device__ BFloat16 operator+(int a, BFloat16 b)
+{
+  return static_cast<BFloat16>(static_cast<float>(a)) + b;
+}
+inline __host__ __device__ BFloat16 operator-(int a, BFloat16 b)
+{
+  return static_cast<BFloat16>(static_cast<float>(a)) - b;
+}
+inline __host__ __device__ BFloat16 operator*(int a, BFloat16 b)
+{
+  return static_cast<BFloat16>(static_cast<float>(a)) * b;
+}
+inline __host__ __device__ BFloat16 operator/(int a, BFloat16 b)
+{
+  return static_cast<BFloat16>(static_cast<float>(a)) / b;
+}
+
+// Comparison — for std::min/std::max
+inline __host__ __device__ bool operator>(BFloat16& lhs, BFloat16& rhs)
+{
+  return float(lhs) > float(rhs);
+}
+inline __host__ __device__ bool operator<(BFloat16& lhs, BFloat16& rhs)
+{
+  return float(lhs) < float(rhs);
+}
+
+// NVBench type registration
+NVBENCH_DECLARE_TYPE_STRINGS(BFloat16, "bf16", "BFloat16");

cccl_upstream/cub/benchmarks/bench/transform/applications/P1/pytorch.cu

@@ -0,0 +1,987 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// ============================================================================
+// NVBench benchmarks for chained elementwise operations was put together by Matthias Jouanneaux (DevTech). It mimics
+// how pytorch uses element-wise kernels (i.e. cub::DeviceTransform) and also tries to preserve pytorch's operators and
+// utility types. The main difference to ordinary CCCL benchmarks is the chaining of several operations in the
+// benchmark's critical section. Furthermore, there are 4 types of work loads covering the combinations of few vs. many
+// input buffers and few vs. many instructions in the kernel. For more discussion see:
+// https://github.com/NVIDIA-dev/cccl_private/issues/639
+// ============================================================================
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include <thrust/device_vector.h>
+
+#include <cuda/iterator>
+#include <cuda/random>
+#include <cuda/std/algorithm.max.h>
+#include <cuda/std/algorithm.min.h>
+#include <cuda/std/algorithm.transform.h>
+#include <cuda/std/cmath>
+#include <cuda/std/execution>
+#include <cuda/std/random>
+#include <cuda/std/type_traits>
+
+#include "../../common.h"
+#include "bfloat16.h"
+
+// ============================================================================
+// at::opmath_type<T> — the compute type for intermediate math
+// float for both float and BFloat16 (ATen/OpMathType.h)
+// ============================================================================
+
+template <typename T>
+struct opmath_type_impl
+{
+  using type = T;
+};
+template <>
+struct opmath_type_impl<BFloat16>
+{
+  using type = float;
+};
+template <typename T>
+using opmath_type = typename opmath_type_impl<T>::type;
+
+// ============================================================================
+// Replicate ATen/c10 helpers without ATen dependencies.
+// Each wrapper is annotated with the ATen source it replicates.
+// ============================================================================
+
+// c10::div_floor_floating (c10/util/generic_math.h:34)
+template <typename scalar_t>
+__device__ __forceinline__ scalar_t div_floor_floating(scalar_t a, scalar_t b)
+{
+  if (b == 0)
+  {
+    return a / b;
+  }
+
+  auto mod = std::fmod(a, b);
+  auto div = (a - mod) / b;
+  if ((mod != 0) && (b < 0) != (mod < 0))
+  {
+    div -= scalar_t(1);
+  }
+
+  scalar_t floordiv;
+  if (div != 0)
+  {
+    floordiv = std::floor(div);
+    if (div - floordiv > scalar_t(0.5))
+    {
+      floordiv += scalar_t(1.0);
+    }
+  }
+  else
+  {
+    floordiv = ::copysignf(scalar_t(0), a / b);
+  }
+  return floordiv;
+}
+
+// is_lerp_weight_small + lerp (native/Lerp.h:11,21)
+template <typename scalar_t>
+__device__ __forceinline__ bool is_lerp_weight_small(scalar_t weight)
+{
+  return std::abs(weight) < scalar_t(0.5);
+}
+
+template <typename scalar_t, typename weight_t>
+__device__ __forceinline__ scalar_t aten_lerp(scalar_t self_, scalar_t end_, weight_t weight_)
+{
+  using opmath_t        = opmath_type<scalar_t>;
+  using opmath_weight_t = opmath_type<weight_t>;
+
+  opmath_t self          = self_;
+  opmath_t end           = end_;
+  opmath_weight_t weight = weight_;
+
+  return is_lerp_weight_small(weight) ? self + weight * (end - self) : end - (end - self) * (opmath_t(1) - weight);
+}
+
+// pointwise_op_impl (native/cuda/DeviceAddCmulCdiv.cuh:9)
+template <typename opmath_t, typename Op>
+__device__ __forceinline__ opmath_t
+pointwise_op_impl(opmath_t input, opmath_t tensor1, opmath_t tensor2, opmath_t alpha, Op op)
+{
+  if (alpha == opmath_t(1))
+  {
+    if constexpr (std::is_same_v<Op, std::multiplies<opmath_t>> && std::is_floating_point_v<opmath_t>)
+    {
+      return std::fma(tensor1, tensor2, input);
+    }
+    else
+    {
+      return input + op(tensor1, tensor2);
+    }
+  }
+  if constexpr (std::is_floating_point_v<opmath_t>)
+  {
+    return std::fma(alpha, op(tensor1, tensor2), input);
+  }
+  else
+  {
+    return input + alpha * op(tensor1, tensor2);
+  }
+}
+
+// DivFunctor (native/cuda/BinaryInternal.h:20)
+template <typename scalar_t>
+struct DivFunctor
+{
+  __device__ scalar_t operator()(scalar_t a, scalar_t b) const
+  {
+    return a / b;
+  }
+};
+
+// MulFunctor (native/cuda/BinaryInternal.h:27)
+template <typename T>
+struct MulFunctor
+{
+  __device__ T operator()(T a, T b) const
+  {
+    return a * b;
+  }
+};
+
+// CUDAFunctorOnSelf_add — torchgen-generated ufunc functor for add(tensor, scalar)
+// (torchgen/dest/ufunc.py, native/ufunc/add.h:14)
+template <typename scalar_t>
+struct CUDAFunctorOnSelf_add
+{
+  using opmath_t = opmath_type<scalar_t>;
+  opmath_t other_;
+  opmath_t alpha_;
+  CUDAFunctorOnSelf_add(opmath_t other, opmath_t alpha)
+      : other_(other)
+      , alpha_(alpha)
+  {}
+  __device__ scalar_t operator()(scalar_t self) const
+  {
+    return static_cast<opmath_t>(self) + alpha_ * other_;
+  }
+};
+
+// CUDAFunctor_add — torchgen-generated ufunc functor for add(tensor, tensor)
+// (torchgen/dest/ufunc.py, native/ufunc/add.h:14)
+template <typename scalar_t>
+struct CUDAFunctor_add
+{
+  using opmath_t = opmath_type<scalar_t>;
+  opmath_t alpha_;
+  CUDAFunctor_add(opmath_t alpha)
+      : alpha_(alpha)
+  {}
+  __device__ scalar_t operator()(scalar_t self, scalar_t other) const
+  {
+    return static_cast<opmath_t>(self) + alpha_ * static_cast<opmath_t>(other);
+  }
+};
+
+// AbsFunctor (native/cuda/AbsKernel.cu:11)
+template <typename scalar_t>
+struct AbsFunctor
+{
+  __device__ __forceinline__ scalar_t operator()(const scalar_t a) const
+  {
+    return std::abs(a);
+  }
+};
+
+// CompareFunctor (native/cuda/CompareKernels.cu:14 / 17)
+enum class OpType
+{
+  GE,
+  GT,
+  LE,
+  LT
+};
+
+template <typename scalar_t>
+struct CompareFunctor
+{
+  constexpr CompareFunctor(OpType op)
+      : op_(op) {};
+  OpType op_;
+  __device__ __forceinline__ bool operator()(scalar_t a, scalar_t b) const
+  {
+    if (op_ == OpType::GE)
+    {
+      return a >= b;
+    }
+    else if (op_ == OpType::GT)
+    {
+      return a > b;
+    }
+    else if (op_ == OpType::LE)
+    {
+      return a <= b;
+    }
+    else
+    { // LT
+      return a < b;
+    }
+  }
+};
+
+// ============================================================================
+// RNG helpers
+// ============================================================================
+
+template <typename T>
+struct normal_gen
+{
+  float mean, stddev;
+  int64_t offset;
+  __host__ __device__ T operator()(int64_t idx) const
+  {
+    cuda::pcg64 rng(42);
+    rng.discard(offset + idx);
+    cuda::std::normal_distribution<float> dist(mean, stddev);
+    return T(dist(rng));
+  }
+};
+
+template <typename T>
+void fill_normal(thrust::device_vector<T>& v, int64_t n, int buf_idx)
+{
+  v.resize(n);
+  cuda::std::transform(
+    cuda::execution::gpu,
+    cuda::counting_iterator<int64_t, int64_t>(0),
+    cuda::counting_iterator<int64_t, int64_t>(n),
+    v.begin(),
+    normal_gen<T>{0.0f, 1.0f, buf_idx * n});
+}
+
+// ============================================================================
+// Helper to call DeviceTransform::Transform with the tuning policy
+// ============================================================================
+
+template <typename... Inputs, typename Output, typename TransformOp>
+void transform(cuda::std::tuple<Inputs...> inputs, Output output, int64_t n, TransformOp op, cudaStream_t stream)
+{
+  auto env = cuda::std::execution::env{
+    cuda::stream_ref{stream}
+#if !TUNE_BASE
+    ,
+    cuda::execution::tune(policy_selector{})
+#endif // !TUNE_BASE
+  };
+  cub::DeviceTransform::Transform(inputs, output, n, op, env);
+}
+
+template <typename Input, typename Output, typename TransformOp>
+void transform(Input input, Output output, int64_t n, TransformOp op, cudaStream_t stream)
+{
+  transform(cuda::std::make_tuple(input), output, n, op, stream);
+}
+
+// ============================================================================
+// Element types
+// ============================================================================
+
+#ifdef TUNE_T
+using element_types = nvbench::type_list<TUNE_T>;
+#else
+using element_types = nvbench::type_list<float, BFloat16>;
+#endif
+
+// ============================================================================
+// many_inputs_many_instructions
+//
+// div_floor -> div_trunc -> div -> atan2 -> hypot ->
+// xlogy -> xlog1py -> logaddexp -> logaddexp2 -> pow
+// 11 inputs, 10 binary ops
+// ============================================================================
+
+template <typename T>
+static void many_inputs_many_instructions(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+
+  constexpr int num_in = 11;
+  thrust::device_vector<T> in[num_in];
+  for (int i = 0; i < num_in; i++)
+  {
+    fill_normal(in[i], n, i);
+  }
+  thrust::device_vector<T> tmpA(n, thrust::no_init), tmpB(n, thrust::no_init);
+
+  T* d_in[num_in];
+  for (int i = 0; i < num_in; i++)
+  {
+    d_in[i] = thrust::raw_pointer_cast(in[i].data());
+  }
+  T* d_a = thrust::raw_pointer_cast(tmpA.data());
+  T* d_b = thrust::raw_pointer_cast(tmpB.data());
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(20L * n);
+  state.add_global_memory_writes<T>(10L * n);
+
+  // logaddexp2 captures inv_log_2 — native/cuda/LogAddExpKernel.cu:272
+  using opmath_t       = opmath_type<T>;
+  const auto inv_log_2 = static_cast<opmath_t>(1.0 / 0.693147180559945309417232121458176);
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](const nvbench::launch& launch) {
+    const auto s = launch.get_stream().get_stream();
+
+    // div_floor: native/cuda/BinaryDivFloorKernel.cu:72, helper c10/util/generic_math.h:34
+    transform(
+      cuda::std::make_tuple(d_in[0], d_in[1]),
+      d_a,
+      n,
+      [] __device__(T a, T b) -> T {
+        return div_floor_floating(a, b);
+      },
+      s);
+
+    // div_trunc: native/cuda/BinaryDivTruncKernel.cu:42
+    transform(
+      cuda::std::make_tuple(d_a, d_in[2]),
+      d_b,
+      n,
+      [] __device__(T a, T b) -> T {
+        return std::trunc(a / b);
+      },
+      s);
+
+    // div: native/cuda/BinaryDivTrueKernel.cu:54, DivFunctor in native/cuda/BinaryInternal.h:20
+    transform(cuda::std::make_tuple(d_b, d_in[3]), d_a, n, DivFunctor<T>(), s);
+
+    // atan2: native/cuda/BinaryGeometricKernels.cu:18
+    transform(
+      cuda::std::make_tuple(d_a, d_in[4]),
+      d_b,
+      n,
+      [] __device__(T a, T b) -> T {
+        return ::atan2(a, b);
+      },
+      s);
+
+    // hypot: native/cuda/BinaryGeometricKernels.cu:29
+    transform(
+      cuda::std::make_tuple(d_b, d_in[5]),
+      d_a,
+      n,
+      [] __device__(T a, T b) -> T {
+        return ::hypot(a, b);
+      },
+      s);
+
+    // xlogy: native/cuda/BinaryMiscOpsKernels.cu:46
+    transform(
+      cuda::std::make_tuple(d_a, d_in[6]),
+      d_b,
+      n,
+      [] __device__(T x, T y) -> T {
+        if (::isnan(static_cast<float>(y)))
+        {
+          return NAN;
+        }
+        if (x == 0)
+        {
+          return 0;
+        }
+        return x * std::log(y);
+      },
+      s);
+
+    // xlog1py: native/cuda/BinaryMiscOpsKernels.cu:60
+    transform(
+      cuda::std::make_tuple(d_b, d_in[7]),
+      d_a,
+      n,
+      [] __device__(T x, T y) -> T {
+        if (::isnan(static_cast<float>(y)))
+        {
+          return NAN;
+        }
+        if (x == 0)
+        {
+          return 0;
+        }
+        return x * std::log1p(y);
+      },
+      s);
+
+    // logaddexp: native/cuda/LogAddExpKernel.cu:253
+    transform(
+      cuda::std::make_tuple(d_a, d_in[8]),
+      d_b,
+      n,
+      [] __device__(T a_, T b_) -> T {
+        using opmath_t = opmath_type<T>;
+        const auto a   = static_cast<opmath_t>(a_);
+        const auto b   = static_cast<opmath_t>(b_);
+        if (::isinf(a) && a == b)
+        {
+          return a;
+        }
+        else
+        {
+          const auto m = ::max(a, b);
+          return m + ::log1p(::exp(-::abs(a - b)));
+        }
+      },
+      s);
+
+    // logaddexp2: native/cuda/LogAddExpKernel.cu:272
+    transform(
+      cuda::std::make_tuple(d_b, d_in[9]),
+      d_a,
+      n,
+      [inv_log_2] __device__(T a_, T b_) -> T {
+        using opmath_t = opmath_type<T>;
+        const auto a   = static_cast<opmath_t>(a_);
+        const auto b   = static_cast<opmath_t>(b_);
+        if (::isinf(a) && a == b)
+        {
+          return a;
+        }
+        else
+        {
+          const auto m = ::max(a, b);
+          return m + ::log1p(::exp2(-::abs(a - b))) * inv_log_2;
+        }
+      },
+      s);
+
+    // pow (tensor,tensor): native/cuda/PowKernel.cu:136, helper native/cuda/Pow.cuh:40
+    transform(
+      cuda::std::make_tuple(d_a, d_in[10]),
+      d_b,
+      n,
+      [] __device__(T base, T exp) -> T {
+        return cuda::std::pow(base, exp);
+      },
+      s);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+// ============================================================================
+// many_inputs_few_instructions
+//
+// mse_loss -> smooth_l1_loss -> huber_loss -> clamp_min ->
+// mul -> add -> addcmul -> lerp(scalar) -> lerp(tensor) -> greater
+// 13 inputs, 8 binary ops + 2 ternary ops
+// ============================================================================
+
+template <typename T>
+static void many_inputs_few_instructions(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+
+  constexpr int num_in = 13;
+  thrust::device_vector<T> in[num_in];
+  for (int i = 0; i < num_in; i++)
+  {
+    fill_normal(in[i], n, i);
+  }
+  thrust::device_vector<T> tmpA(n, thrust::no_init), tmpB(n, thrust::no_init);
+
+  T* d_in[num_in];
+  for (int i = 0; i < num_in; i++)
+  {
+    d_in[i] = thrust::raw_pointer_cast(in[i].data());
+  }
+  T* d_a = thrust::raw_pointer_cast(tmpA.data());
+  T* d_b = thrust::raw_pointer_cast(tmpB.data());
+
+  // 8 binary (16 reads) + 2 ternary (6 reads) = 22 reads, 10 writes
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(22L * n);
+  state.add_global_memory_writes<T>(10L * n);
+
+  // Captured scalar parameters, matching how ATen sets them up before gpu_kernel
+  using opmath_t = opmath_type<T>;
+  T beta_val(1.0); // smooth_l1: scalar_t beta_val(beta)
+  T delta_val(1.0); // huber: scalar_t delta_val(delta)
+  // note: opmath_type is same as at::acc_type<scalar_t, true> here
+  using accscalar_t     = opmath_type<T>; // addcmul: at::acc_type<scalar_t, true>
+  const auto alpha      = accscalar_t(1); // addcmul: value.to<accscalar_t>()
+  const auto weight_val = opmath_t(4.0); // lerp scalar: weight.to<opmath_t>()
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](const nvbench::launch& launch) {
+    const auto s = launch.get_stream().get_stream();
+
+    // mse_loss: native/cuda/BinaryMiscOpsKernels.cu:37
+    transform(
+      cuda::std::make_tuple(d_in[0], d_in[1]),
+      d_a,
+      n,
+      [] __device__(T a, T b) -> T {
+        auto diff = a - b;
+        return diff * diff;
+      },
+      s);
+
+    // smooth_l1_loss(beta=1.0): native/cuda/BinaryMiscOpsKernels.cu:19
+    transform(
+      cuda::std::make_tuple(d_a, d_in[2]),
+      d_b,
+      n,
+      [beta_val] __device__(T a, T b) -> T {
+        auto z = ::abs(a - b);
+        return z < beta_val ? T(0.5) * z * z / beta_val : z - T(0.5) * beta_val;
+      },
+      s);
+
+    // huber_loss(delta=1.0): native/cuda/BinaryMiscOpsKernels.cu:29
+    transform(
+      cuda::std::make_tuple(d_b, d_in[3]),
+      d_a,
+      n,
+      [delta_val] __device__(T a, T b) -> T {
+        auto z = ::abs(a - b);
+        return z < delta_val ? T(0.5) * z * z : delta_val * (z - T(0.5) * delta_val);
+      },
+      s);
+
+    // clamp(min=tensor) -> maximum: native/cuda/MaxMinElementwiseKernel.cu:28
+    transform(
+      cuda::std::make_tuple(d_a, d_in[4]),
+      d_b,
+      n,
+      [] __device__(T a, T b) -> T {
+        if (a != a)
+        {
+          return a;
+        }
+        else if (b != b)
+        {
+          return b;
+        }
+        else
+        {
+          return ::max(a, b);
+        }
+      },
+      s);
+
+    // mul: native/cuda/BinaryMulKernel.cu:39, MulFunctor in native/cuda/BinaryInternal.h:27
+    using mul_opmath_t = opmath_type<T>;
+    transform(cuda::std::make_tuple(d_b, d_in[5]), d_a, n, MulFunctor<mul_opmath_t>(), s);
+
+    // add(alpha=1): native/ufunc/add.h:14, torchgen/dest/ufunc.py
+    transform(cuda::std::make_tuple(d_a, d_in[6]), d_b, n, CUDAFunctor_add<T>(1.0), s);
+
+    // addcmul(value=1): native/cuda/PointwiseOpsKernel.cu:87, native/cuda/DeviceAddCmulCdiv.cuh:9
+    transform(
+      cuda::std::make_tuple(d_b, d_in[7], d_in[8]),
+      d_a,
+      n,
+      [alpha] __device__(T a, T b, T c) -> T {
+        return pointwise_op_impl<accscalar_t>(a, b, c, alpha, cuda::std::multiplies<accscalar_t>());
+      },
+      s);
+
+    // lerp(weight=4.0): native/cuda/Lerp.cu:130, native/Lerp.h:21
+    transform(
+      cuda::std::make_tuple(d_a, d_in[9]),
+      d_b,
+      n,
+      [=] __device__(T self_val, T end_val) {
+        return aten_lerp(self_val, end_val, weight_val);
+      },
+      s);
+
+    // lerp(weight=tensor): native/cuda/Lerp.cu:76, native/Lerp.h:21
+    transform(
+      cuda::std::make_tuple(d_b, d_in[10], d_in[11]),
+      d_a,
+      n,
+      [] __device__(T self_val, T end_val, T weight_val) -> T {
+        return aten_lerp(self_val, end_val, weight_val);
+      },
+      s);
+
+    // note: even though output is bool, we use d_b as output because
+    // it must hold at least enough memory per element for bool (1 byte)
+    // greater: native/cuda/CompareKernels.cu:69
+    CompareFunctor<T> comp_f(OpType::GT);
+    transform(cuda::std::make_tuple(d_a, d_in[12]), d_b, n, comp_f, s);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+// ============================================================================
+// few_inputs_many_instructions
+//
+// pow(2.5) -> tanh -> sin -> cos -> softplus ->
+// silu -> mish -> elu -> gelu -> logsigmoid
+// 1 input, 10 unary ops
+// ============================================================================
+
+template <typename T>
+static void few_inputs_many_instructions(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+
+  thrust::device_vector<T> input(n, thrust::no_init);
+  fill_normal(input, n, 0);
+  thrust::device_vector<T> tmpA(n, thrust::no_init), tmpB(n, thrust::no_init);
+
+  T* d_in = thrust::raw_pointer_cast(input.data());
+  T* d_a  = thrust::raw_pointer_cast(tmpA.data());
+  T* d_b  = thrust::raw_pointer_cast(tmpB.data());
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(10L * n);
+  state.add_global_memory_writes<T>(10L * n);
+
+  // Captured scalar parameters
+  using opmath_t        = opmath_type<T>;
+  const auto exp_val    = T(2.5); // pow: exp_scalar.to<scalar_t>()
+  const auto beta       = opmath_t(1); // softplus: beta_.to<opmath_t>()
+  const auto threshold  = opmath_t(20); // softplus: threshold_.to<opmath_t>()
+  const auto negcoef    = opmath_t(1) * opmath_t(1); // elu: alpha * scale
+  const auto poscoef    = opmath_t(1); // elu: scale
+  const auto negiptcoef = opmath_t(1); // elu: input_scale
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](const nvbench::launch& launch) {
+    const auto s = launch.get_stream().get_stream();
+
+    // pow(scalar=2.5): native/cuda/PowKernel.cu:163, helper native/cuda/Pow.cuh:40
+    transform(
+      d_in,
+      d_a,
+      n,
+      [=] __device__(T base) -> T {
+        return cuda::std::pow(base, exp_val);
+      },
+      s);
+
+    // tanh: native/cuda/UnaryGeometricTanhKernel.cu:50
+    transform(
+      d_a,
+      d_b,
+      n,
+      [] __device__(T a) -> T {
+        return ::tanh(a);
+      },
+      s);
+
+    // sin: native/cuda/UnaryGeometricSinKernel.cu:50
+    transform(
+      d_b,
+      d_a,
+      n,
+      [] __device__(T a) -> T {
+        return ::sin(a);
+      },
+      s);
+
+    // cos: native/cuda/UnaryGeometricCosKernel.cu:50
+    transform(
+      d_a,
+      d_b,
+      n,
+      [] __device__(T a) -> T {
+        return ::cos(a);
+      },
+      s);
+
+    // softplus(beta=1, threshold=20): native/cuda/ActivationSoftplusKernel.cu:35
+    transform(
+      d_b,
+      d_a,
+      n,
+      [beta, threshold] __device__(T a) -> T {
+        using opmath_t = opmath_type<T>;
+        opmath_t aop   = static_cast<opmath_t>(a);
+        return (aop * beta) > threshold ? aop : (::log1p(std::exp(aop * beta))) / beta;
+      },
+      s);
+
+    // silu: native/cuda/ActivationSiluKernel.cu:30
+    transform(
+      d_a,
+      d_b,
+      n,
+      [] __device__(T x) -> T {
+        using opmath_t       = opmath_type<T>;
+        const opmath_t x_acc = static_cast<opmath_t>(x);
+        return x_acc / (opmath_t(1) + ::exp(-x_acc));
+      },
+      s);
+
+    // mish: native/cuda/ActivationMishKernel.cu:29
+    transform(
+      d_b,
+      d_a,
+      n,
+      [] __device__(T x) -> T {
+        using opmath_t       = opmath_type<T>;
+        const opmath_t x_acc = static_cast<opmath_t>(x);
+        return x_acc * ::tanhf(::log1pf(::expf(x_acc)));
+      },
+      s);
+
+    // elu(alpha=1, scale=1, input_scale=1): native/cuda/ActivationEluKernel.cu:37
+    transform(
+      d_a,
+      d_b,
+      n,
+      [negcoef, poscoef, negiptcoef] __device__(T a) -> T {
+        using opmath_t = opmath_type<T>;
+        opmath_t aop   = static_cast<opmath_t>(a);
+        return aop > 0 ? aop * poscoef : std::expm1(aop * negiptcoef) * negcoef;
+      },
+      s);
+
+    // gelu(approximate='none'): native/cuda/ActivationGeluKernel.cu:35
+    transform(
+      d_b,
+      d_a,
+      n,
+      [] __device__(T x) -> T {
+        using opmath_t            = opmath_type<T>;
+        constexpr opmath_t kAlpha = M_SQRT1_2;
+        return static_cast<opmath_t>(x) * opmath_t(0.5) * (opmath_t(1) + ::erf(static_cast<opmath_t>(x) * kAlpha));
+      },
+      s);
+
+    // logsigmoid: native/cuda/ActivationLogSigmoidKernel.cu:30
+    transform(
+      d_a,
+      d_b,
+      n,
+      [] __device__(T in_) -> T {
+        using opmath_t    = opmath_type<T>;
+        const opmath_t in = in_;
+        const auto min    = cuda::std::min(opmath_t(0), in);
+        const auto z      = std::exp(-std::abs(in));
+        return min - std::log1p(z);
+      },
+      s);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+// ============================================================================
+// few_inputs_few_instructions
+//
+// add(0.5) -> neg -> clamp(-2,1) -> abs -> mul(1.5) ->
+// leaky_relu -> hardswish -> hardshrink -> hardsigmoid -> gt(0)
+// 1 input, 10 unary ops
+// ============================================================================
+
+template <typename T>
+static void few_inputs_few_instructions(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+
+  thrust::device_vector<T> input(n, thrust::no_init);
+  fill_normal(input, n, 0);
+  thrust::device_vector<T> tmpA(n, thrust::no_init), tmpB(n, thrust::no_init);
+
+  T* d_in = thrust::raw_pointer_cast(input.data());
+  T* d_a  = thrust::raw_pointer_cast(tmpA.data());
+  T* d_b  = thrust::raw_pointer_cast(tmpB.data());
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(10L * n);
+  state.add_global_memory_writes<T>(10L * n);
+
+  // Captured scalar parameters
+  using opmath_t = opmath_type<T>;
+
+  // clamp: native/cuda/TensorCompare.cu:58
+  const auto lim0_val = opmath_t(-2);
+  const auto lim1_val = opmath_t(1);
+  const auto minmax   = 2; // 0=Min, 1=Max, 2=MinMax
+
+  // mul scalar: MulFunctor via BUnaryFunctor with captured scalar
+  const auto mul_scalar = opmath_t(1.5);
+
+  // leaky_relu: native/cuda/ActivationLeakyReluKernel.cu:31
+  const auto negval = opmath_t(0.01); // negval_.to<opmath_t>()
+
+  // hardswish: native/cuda/ActivationHardswishKernel.cu:25
+  const opmath_t zero(0.0f);
+  const opmath_t one_sixth(1.0f / 6.0f);
+  const opmath_t three(3.0f);
+  const opmath_t six(6.0f);
+
+  // hardshrink: native/cuda/ActivationHardshrinkKernel.cu:29
+  const auto lambd = T(0.5); // value.to<scalar_t>()
+
+  // gt scalar: native/cuda/CompareKernels.cu:47
+  const T rhs(0);
+
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](const nvbench::launch& launch) {
+    const auto s = launch.get_stream().get_stream();
+
+    // add(scalar, alpha=1): native/ufunc/add.h:14
+    transform(d_in, d_a, n, CUDAFunctorOnSelf_add<T>(T(0.5), T(1)), s);
+
+    // neg: native/cuda/UnarySignKernels.cu:54
+    transform(
+      d_a,
+      d_b,
+      n,
+      [] __device__(T a) -> T {
+        return -a;
+      },
+      s);
+
+    // clamp(min=-2, max=1): native/cuda/TensorCompare.cu:58 (MinMax branch)
+    transform(
+      d_b,
+      d_a,
+      n,
+      [=] __device__(T v) -> T {
+        using opmath_t = opmath_type<T>;
+        if (::isnan(static_cast<opmath_t>(v)))
+        {
+          return v;
+        }
+        else if (minmax == 0)
+        {
+          return ::max(static_cast<opmath_t>(v), lim0_val);
+        }
+        else if (minmax == 1)
+        {
+          return ::min(static_cast<opmath_t>(v), lim0_val);
+        }
+        else
+        {
+          return ::min(::max(static_cast<opmath_t>(v), lim0_val), lim1_val);
+        }
+      },
+      s);
+
+    // abs: native/cuda/AbsKernel.cu:39, AbsFunctor:11
+    transform(d_a, d_b, n, AbsFunctor<T>(), s);
+
+    // mul(scalar=1.5): native/cuda/BinaryMulKernel.cu:39, MulFunctor via BUnaryFunctor
+    transform(
+      d_b,
+      d_a,
+      n,
+      [mul_scalar] __device__(T a) -> T {
+        return MulFunctor<opmath_t>()(a, mul_scalar);
+      },
+      s);
+
+    // leaky_relu(slope=0.01): native/cuda/ActivationLeakyReluKernel.cu:31
+    transform(
+      d_a,
+      d_b,
+      n,
+      [negval] __device__(T a) -> T {
+        using opmath_t = opmath_type<T>;
+        opmath_t aop   = static_cast<opmath_t>(a);
+        return aop > opmath_t(0) ? aop : aop * negval;
+      },
+      s);
+
+    // hardswish: native/cuda/ActivationHardswishKernel.cu:25
+    transform(
+      d_b,
+      d_a,
+      n,
+      [zero, one_sixth, three, six] __device__(T self_val) -> T {
+        using opmath_t = opmath_type<T>;
+        opmath_t x     = static_cast<opmath_t>(self_val);
+        return x * cuda::std::min(cuda::std::max(x + three, zero), six) * one_sixth;
+      },
+      s);
+
+    // hardshrink(lambd=0.5): native/cuda/ActivationHardshrinkKernel.cu:29
+    transform(
+      d_a,
+      d_b,
+      n,
+      [lambd] __device__(T a) -> T {
+        return (a >= -lambd && a <= lambd) ? T(0) : a;
+      },
+      s);
+
+    // hardsigmoid: native/cuda/ActivationHardsigmoidKernel.cu:30
+    transform(
+      d_b,
+      d_a,
+      n,
+      [zero, one_sixth, three, six] __device__(T self_val) -> T {
+        using opmath_t = opmath_type<T>;
+        opmath_t x     = static_cast<opmath_t>(self_val);
+        return cuda::std::min<opmath_t>(cuda::std::max<opmath_t>(x + three, zero), six) * one_sixth;
+      },
+      s);
+
+    // gt(scalar=0): native/cuda/CompareKernels.cu:47
+    CompareFunctor<T> comp_f(OpType::GT);
+    transform(
+      d_a,
+      d_b,
+      n,
+      [=] __device__(T lhs) -> T {
+        return comp_f(lhs, rhs);
+      },
+      s);
+  });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(many_inputs_many_instructions, NVBENCH_TYPE_AXES(element_types))
+  .set_name("many_inputs_many_instructions")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));
+
+NVBENCH_BENCH_TYPES(many_inputs_few_instructions, NVBENCH_TYPE_AXES(element_types))
+  .set_name("many_inputs_few_instructions")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));
+
+NVBENCH_BENCH_TYPES(few_inputs_many_instructions, NVBENCH_TYPE_AXES(element_types))
+  .set_name("few_inputs_many_instructions")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));
+
+NVBENCH_BENCH_TYPES(few_inputs_few_instructions, NVBENCH_TYPE_AXES(element_types))
+  .set_name("few_inputs_few_instructions")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));

cccl_upstream/cub/benchmarks/bench/transform/babelstream.cu

@@ -0,0 +1,191 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+#ifdef TUNE_T
+using element_types = nvbench::type_list<TUNE_T>;
+#else
+using element_types =
+  nvbench::type_list<std::int8_t,
+                     std::int16_t,
+                     float,
+                     double
+#  if _CCCL_HAS_INT128()
+                     ,
+                     __int128
+#  endif
+                     >;
+#endif
+
+// BabelStream uses 2^25, H200 can fit 2^31 int128s
+// 2^20 chars / 2^16 int128 saturate V100 (min_bytes_in_flight =12 * SM count =80)
+// 2^21 chars / 2^17 int128 saturate A100 (min_bytes_in_flight =16 * SM count =108)
+// 2^23 chars / 2^19 int128 saturate H100/H200 HBM3 (min_bytes_in_flight =32or48 * SM count =132)
+// inline auto array_size_powers = std::vector<nvbench::int64_t>{28};
+inline auto array_size_powers = nvbench::range(16, 32, 4);
+
+// Modified from BabelStream to also work for integers and to make nstream maintain a consistent workload since it
+// overwrites one input array. If the data changed at each iteration, the performance would be unstable.
+inline constexpr auto startA      = 11; // BabelStream: 0.1
+inline constexpr auto startB      = 2; // BabelStream: 0.2
+inline constexpr auto startC      = 1; // BabelStream: 0.1
+inline constexpr auto startScalar = -2; // BabelStream: 0.4
+
+static_assert(startA == (startA + startB + startScalar * startC), "nstream must have a consistent workload");
+
+template <typename T>
+static void mul(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(n);
+
+  const T scalar = startScalar;
+  bench_transform(
+    state, cuda::std::tuple{c.begin() + unaligned}, b.begin() + unaligned, n, [=] _CCCL_DEVICE(const T& ci) {
+      return ci * scalar;
+    });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(mul, NVBENCH_TYPE_AXES(element_types))
+  .set_name("mul")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);
+
+template <typename T>
+static void add(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+
+  thrust::device_vector<T> a(n + unaligned, startA);
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(
+    state,
+    cuda::std::tuple{a.begin() + unaligned, b.begin() + unaligned},
+    c.begin() + unaligned,
+    n,
+    [] _CCCL_DEVICE(const T& ai, const T& bi) -> T {
+      return ai + bi;
+    });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(add, NVBENCH_TYPE_AXES(element_types))
+  .set_name("add")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);
+
+template <typename T>
+static void triad(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+
+  thrust::device_vector<T> a(n + unaligned, startA);
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+  const T scalar = startScalar;
+  bench_transform(
+    state,
+    cuda::std::tuple{b.begin() + unaligned, c.begin() + unaligned},
+    a.begin() + unaligned,
+    n,
+    [=] _CCCL_DEVICE(const T& bi, const T& ci) {
+      return bi + scalar * ci;
+    });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(triad, NVBENCH_TYPE_AXES(element_types))
+  .set_name("triad")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);
+
+template <typename T>
+static void nstream(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+
+  thrust::device_vector<T> a(n + unaligned, startA);
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(3 * n);
+  state.add_global_memory_writes<T>(n);
+  const T scalar = startScalar;
+  bench_transform(
+    state,
+    cuda::std::tuple{a.begin() + unaligned, b.begin() + unaligned, c.begin() + unaligned},
+    a.begin() + unaligned,
+    n,
+    [=] _CCCL_DEVICE(const T& ai, const T& bi, const T& ci) {
+      return ai + bi + scalar * ci;
+    });
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(nstream, NVBENCH_TYPE_AXES(element_types))
+  .set_name("nstream")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);

cccl_upstream/cub/benchmarks/bench/transform/common.h

@@ -0,0 +1,96 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+#pragma once
+
+// keep checks at the top so compilation of discarded variants fails really fast
+#include <cub/device/dispatch/dispatch_transform.cuh>
+#if !TUNE_BASE
+#  if _CCCL_PP_COUNT(__CUDA_ARCH_LIST__) != 1
+#    error "When tuning, this benchmark does not support being compiled for multiple architectures"
+#  endif
+#  if TUNE_ALGORITHM == 3
+#    if (__CUDA_ARCH_LIST__) < 900
+#      error "Cannot compile algorithm 3 (ublkcp) below sm90"
+#    endif
+#  endif // TUNE_ALGORITHM == 3
+#endif // !TUNE_BASE
+
+#include <cub/util_namespace.cuh>
+
+#include <cuda/__numeric/narrow.h>
+#include <cuda/std/cstdint>
+#include <cuda/std/type_traits>
+
+#include <stdexcept>
+
+#include <nvbench_helper.cuh>
+
+#if !TUNE_BASE
+struct policy_selector
+{
+  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability cc) const -> cub::TransformPolicy
+  {
+    const int min_bytes_in_flight = cub::detail::transform::cc_to_min_bytes_in_flight(cc) + TUNE_BIF_BIAS;
+#  if TUNE_ALGORITHM == 0 || TUNE_ALGORITHM == 1
+    // setup prefetch, since it's either used directly or the fallback to vectorized
+    auto algorithm                = cub::TransformAlgorithm::prefetch;
+    auto pref_policy              = cub::TransformPrefetchPolicy{};
+    pref_policy.threads_per_block = TUNE_THREADS;
+    pref_policy.unroll_factor     = TUNE_UNROLL_FACTOR;
+#    ifdef TUNE_PREFETCH_MULT
+    pref_policy.prefetch_byte_stride = 32 * TUNE_PREFETCH_MULT;
+#    endif //  TUNE_PREFETCH_MULT
+#    ifdef TUNE_ITEMS_PER_THREAD_NO_INPUT
+    pref_policy.items_per_thread_no_input = TUNE_ITEMS_PER_THREAD_NO_INPUT;
+#    endif // TUNE_ITEMS_PER_THREAD_NO_INPUT
+
+    // setup vectorized if requested
+    auto vec_policy = cub::TransformVectorizedPolicy{};
+#    if TUNE_ALGORITHM == 1
+    algorithm                    = cub::TransformAlgorithm::vectorized;
+    vec_policy.threads_per_block = TUNE_THREADS;
+    vec_policy.vec_size          = (1 << TUNE_VEC_SIZE_POW2);
+    vec_policy.items_per_thread  = vec_policy.vec_size * TUNE_UNROLL_FACTOR;
+#    endif
+    return {min_bytes_in_flight, algorithm, pref_policy, vec_policy, {}};
+#  elif TUNE_ALGORITHM == 2
+    constexpr auto algorithm = cub::TransformAlgorithm::ldgsts;
+    auto policy              = cub::TransformAsyncCopyPolicy{};
+    policy.threads_per_block = TUNE_THREADS;
+    policy.unroll_factor     = TUNE_UNROLL_FACTOR;
+    return {min_bytes_in_flight, algorithm, {}, {}, policy};
+#  elif TUNE_ALGORITHM == 3
+    constexpr auto algorithm = cub::TransformAlgorithm::ublkcp;
+    auto policy              = cub::TransformAsyncCopyPolicy{};
+    policy.threads_per_block = TUNE_THREADS;
+    policy.unroll_factor     = TUNE_UNROLL_FACTOR;
+    return {min_bytes_in_flight, algorithm, {}, {}, policy};
+#  else // TUNE_ALGORITHM
+#    error Policy hub does not yet implement the specified value for algorithm
+#  endif // TUNE_ALGORITHM
+  }
+};
+#endif // !TUNE_BASE
+
+template <typename... RandomAccessIteratorsIn, typename RandomAccessIteratorOut, typename TransformOp>
+void bench_transform(nvbench::state& state,
+                     cuda::std::tuple<RandomAccessIteratorsIn...> inputs,
+                     RandomAccessIteratorOut output,
+                     ::cuda::std::int64_t num_items,
+                     TransformOp transform_op)
+{
+  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](const nvbench::launch& launch) {
+    cub::DeviceTransform::Transform(
+      inputs,
+      output,
+      num_items,
+      transform_op,
+      cuda::std::execution::env{::cuda::stream_ref{launch.get_stream().get_stream()}
+#if !TUNE_BASE
+                                ,
+                                cuda::execution::tune(policy_selector{})
+#endif // !TUNE_BASE
+      });
+  });
+}

cccl_upstream/cub/benchmarks/bench/transform/complex_cmp.cu

@@ -0,0 +1,51 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+// This benchmark tests overlapping memory regions for reading and is compute intensive
+
+static void compare_complex(nvbench::state& state)
+try
+{
+  const auto n                        = state.get_int64("Elements{io}");
+  thrust::device_vector<complex32> in = generate(n);
+  thrust::device_vector<bool> out(n - 1);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<complex32>(n);
+  state.add_global_memory_writes<bool>(n);
+
+  // the complex comparison needs lots of compute and transform reads from overlapping input
+  using compare_op = less_t;
+  bench_transform(state, cuda::std::tuple{in.begin(), in.begin() + 1}, out.begin(), n - 1, compare_op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH(compare_complex)
+  .set_name("compare_complex")
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/fib.cu

@@ -0,0 +1,78 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+// This benchmark is compute intensive with diverging threads
+
+template <class IndexT, class OutputT>
+struct fib_t
+{
+  __device__ OutputT operator()(IndexT n)
+  {
+    OutputT t1 = 0;
+    OutputT t2 = 1;
+
+    if (n < 1)
+    {
+      return t1;
+    }
+    if (n == 1)
+    {
+      return t1;
+    }
+    if (n == 2)
+    {
+      return t2;
+    }
+    for (IndexT i = 3; i <= n; ++i)
+    {
+      const auto next = t1 + t2;
+      t1              = t2;
+      t2              = next;
+    }
+    return t2;
+  }
+};
+static void fibonacci(nvbench::state& state)
+try
+{
+  using index_t                     = int64_t;
+  using output_t                    = uint32_t;
+  const auto n                      = state.get_int64("Elements{io}");
+  thrust::device_vector<index_t> in = generate(n, bit_entropy::_1_000, index_t{0}, index_t{42});
+  thrust::device_vector<output_t> out(n);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<index_t>(n);
+  state.add_global_memory_writes<output_t>(n);
+
+  bench_transform(state, cuda::std::tuple{in.begin()}, out.begin(), n, fib_t<index_t, output_t>{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH(fibonacci).set_name("fibonacci").add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/fill.cu

@@ -0,0 +1,64 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// for filling, we can only use the prefetch and the vectorized algorithm
+// %RANGE% TUNE_ALGORITHM alg 0:2:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_ITEMS_PER_THREAD_NO_INPUT ipt 1:32:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_ITEMS_PER_THREAD_NO_INPUT != 1)
+#  error "Non-prefetch algorithms require the no input items per thread to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1 || TUNE_VECTORS_PER_THREAD != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+template <typename T>
+struct return_constant
+{
+  T value;
+
+  _CCCL_DEVICE auto operator()() const -> T
+  {
+    return value;
+  }
+};
+
+template <typename T>
+static void fill(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  // A 32-bit offset type or the value 0 or 0xFF... have <1% performance impact
+  const auto value     = T{42};
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+  thrust::device_vector<T> out(n + unaligned);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(0);
+  state.add_global_memory_writes<T>(n);
+
+  bench_transform(state, cuda::std::tuple{}, out.begin() + unaligned, n, return_constant<T>{value});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(fill, NVBENCH_TYPE_AXES(integral_types))
+  .set_name("fill")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/grayscale.cu

@@ -0,0 +1,96 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+template <typename T>
+struct rgb_t
+{
+  T r;
+  T g;
+  T b;
+
+  __device__ T grayscale() const
+  {
+    static constexpr T w_r(0.2989);
+    static constexpr T w_g(0.587);
+    static constexpr T w_b(0.114);
+
+    return w_r * r + w_g * g + w_b * b;
+  }
+};
+
+template <typename T>
+struct transform_op_t
+{
+  __device__ T operator()(rgb_t<T> pixel) const
+  {
+    return pixel.grayscale();
+  }
+};
+
+template <typename T>
+static void grayscale(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  using pixel_t = rgb_t<T>;
+  const auto n  = state.get_int64("Elements{io}");
+
+  // Generate random RGB data by creating separate R, G, B vectors and combining them
+  thrust::device_vector<T> r_data = generate(n);
+  thrust::device_vector<T> g_data = generate(n);
+  thrust::device_vector<T> b_data = generate(n);
+
+  thrust::device_vector<pixel_t> input(n, thrust::no_init);
+  thrust::transform(
+    thrust::make_zip_iterator(r_data.begin(), g_data.begin(), b_data.begin()),
+    thrust::make_zip_iterator(r_data.end(), g_data.end(), b_data.end()),
+    input.begin(),
+    thrust::make_zip_function([] __device__(T r, T g, T b) {
+      return pixel_t{r, g, b};
+    }));
+
+  thrust::device_vector<T> output(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<pixel_t>(n);
+  state.add_global_memory_writes<T>(n);
+
+  bench_transform(state, cuda::std::tuple{input.begin()}, output.begin(), n, transform_op_t<T>{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+#ifdef TUNE_T
+using value_types = nvbench::type_list<TUNE_T>;
+#else
+using value_types = nvbench::type_list<float, double>;
+#endif
+
+NVBENCH_BENCH_TYPES(grayscale, NVBENCH_TYPE_AXES(value_types))
+  .set_name("grayscale")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/heavy.cu

@@ -0,0 +1,93 @@
+// SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: BSD-3-Clause
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+// This benchmark uses a LOT of registers and is compute intensive.
+
+template <int N>
+struct heavy_functor
+{
+  // we need to use an unsigned type so overflow in arithmetic wraps around
+  __device__ std::uint32_t operator()(std::uint32_t data) const
+  {
+    std::uint32_t reg[N];
+    reg[0] = data;
+    for (int i = 1; i < N; ++i)
+    {
+      reg[i] = reg[i - 1] * reg[i - 1] + 1;
+    }
+    for (int i = 0; i < N; ++i)
+    {
+      reg[i] = (reg[i] * reg[i]) % 19;
+    }
+    for (int i = 0; i < N; ++i)
+    {
+      reg[i] = reg[N - i - 1] * reg[i];
+    }
+    std::uint32_t x = 0;
+    for (int i = 0; i < N; ++i)
+    {
+      x += reg[i];
+    }
+    return x;
+  }
+};
+
+template <typename Heaviness>
+static void heavy(nvbench::state& state, nvbench::type_list<Heaviness>)
+try
+{
+  using value_t = std::uint32_t;
+  const auto n  = state.get_int64("Elements{io}");
+
+  thrust::device_vector<value_t> in = generate(n);
+  thrust::device_vector<value_t> out(n);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<value_t>(n);
+  state.add_global_memory_writes<value_t>(n);
+
+  bench_transform(state, cuda::std::tuple{in.begin()}, out.begin(), n, heavy_functor<Heaviness::value>{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+using ::cuda::std::integral_constant;
+#ifdef TUNE_Heaviness
+using heaviness = nvbench::type_list<TUNE_Heaviness>; // expands to "integral_constant<int, ...>"
+#else
+using heaviness =
+  nvbench::type_list<integral_constant<int, 32>,
+                     integral_constant<int, 64>,
+                     integral_constant<int, 128>,
+                     integral_constant<int, 256>>;
+#endif
+
+NVBENCH_BENCH_TYPES(heavy, NVBENCH_TYPE_AXES(heaviness))
+  .set_name("heavy")
+  .set_type_axes_names({"Heaviness{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/pytorch.cu

@@ -0,0 +1,209 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// %RANGE% TUNE_BIF_BIAS bif -16:16:4
+// %RANGE% TUNE_ALGORITHM alg 0:4:1
+// %RANGE% TUNE_THREADS tpb 128:1024:128
+
+// for TUNE_ALGORITHM == 1 (vectorized), this is the number of vectors per thread, which is similar in spirit
+// %RANGE% TUNE_UNROLL_FACTOR unrl 1:4:1
+
+// those parameters only apply if TUNE_ALGORITHM == 0 (prefetch)
+// %RANGE% TUNE_PREFETCH_MULT pref 1:3:1
+
+// those parameters only apply if TUNE_ALGORITHM == 1 (vectorized)
+// %RANGE% TUNE_VEC_SIZE_POW2 vsp2 1:6:1
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+#  error "Non-prefetch algorithms require prefetch multiple to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 0 && (TUNE_PREFETCH_MULT != 1)
+
+#if !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+#  error "Non-vectorized algorithms require vector size to be 1 since they ignore the parameters"
+#endif // !TUNE_BASE && TUNE_ALGORITHM != 1 && (TUNE_VEC_SIZE_POW2 != 1)
+
+#include "common.h"
+
+#ifdef TUNE_T
+using element_types = nvbench::type_list<TUNE_T>;
+#else
+using element_types = nvbench::type_list<
+#  if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
+  __half,
+#  endif
+#  if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
+  __nv_bfloat16,
+#  endif
+  float>;
+#endif
+
+template <typename Op, typename T>
+static void unary(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+  thrust::device_vector<T> in(n, 1337);
+  thrust::device_vector<T> out(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(n);
+
+  bench_transform(state, cuda::std::tuple{in.begin()}, out.begin(), n, Op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+#define BENCHMARK_UNARY(func)                                               \
+  template <typename T>                                                     \
+  static void func##_bench(nvbench::state& state, nvbench::type_list<T> tl) \
+  {                                                                         \
+    unary<func##_op>(state, tl);                                            \
+  }                                                                         \
+                                                                            \
+  NVBENCH_BENCH_TYPES(func##_bench, NVBENCH_TYPE_AXES(element_types))       \
+    .set_name(#func)                                                        \
+    .set_type_axes_names({"T{ct}"})                                         \
+    .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));
+
+// See: https://github.com/pytorch/pytorch/blob/main/aten/src/ATen/OpMathType.h
+using opmath_t = float;
+
+// See for example:
+// https://github.com/pytorch/pytorch/blob/5a48148c1ab83c1e3779283d904ba5744bbe8eb3/aten/src/ATen/native/cuda/ActivationLeakyReluKernel.cu#L28-L35
+struct relu_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T value) const
+  {
+    return static_cast<T>(static_cast<opmath_t>(value) > opmath_t{0} ? static_cast<opmath_t>(value) : opmath_t{0});
+  }
+};
+BENCHMARK_UNARY(relu);
+
+// See for example:
+// https://github.com/pytorch/pytorch/blob/5a48148c1ab83c1e3779283d904ba5744bbe8eb3/aten/src/ATen/native/cuda/UnarySpecialOpsKernel.cu#L152-L157
+struct sigmoid_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T value) const
+  {
+    return static_cast<T>(opmath_t{1} / (opmath_t{1} + ::cuda::std::exp(-static_cast<opmath_t>(value))));
+  }
+};
+BENCHMARK_UNARY(sigmoid);
+
+struct tanh_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T value) const
+  {
+    return ::cuda::std::tanh(value);
+  }
+};
+BENCHMARK_UNARY(tanh);
+
+// See for example:
+// https://github.com/pytorch/pytorch/blob/5a48148c1ab83c1e3779283d904ba5744bbe8eb3/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L21
+struct gelu_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T value) const
+  {
+    return static_cast<opmath_t>(value) * opmath_t{0.5}
+         * (opmath_t{1} + ::cuda::std::erf(static_cast<opmath_t>(value) * opmath_t{M_SQRT1_2}));
+  }
+};
+BENCHMARK_UNARY(gelu);
+
+struct sin_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T value) const
+  {
+    return ::cuda::std::sin(value);
+  }
+};
+BENCHMARK_UNARY(sin);
+
+struct exp_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T value) const
+  {
+    return ::cuda::std::exp(value);
+  }
+};
+BENCHMARK_UNARY(exp);
+
+template <typename Op, typename T>
+static void binary(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+  thrust::device_vector<T> in1(n, 1337);
+  thrust::device_vector<T> in2(n, 42);
+  thrust::device_vector<T> out(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+
+  bench_transform(state, cuda::std::tuple{in1.begin(), in2.begin()}, out.begin(), n, Op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+#define BENCHMARK_BINARY(func)                                              \
+  template <typename T>                                                     \
+  static void func##_bench(nvbench::state& state, nvbench::type_list<T> tl) \
+  {                                                                         \
+    binary<func##_op>(state, tl);                                           \
+  }                                                                         \
+                                                                            \
+  NVBENCH_BENCH_TYPES(func##_bench, NVBENCH_TYPE_AXES(element_types))       \
+    .set_name(#func)                                                        \
+    .set_type_axes_names({"T{ct}"})                                         \
+    .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 28, 4));
+
+using add_op = cuda::std::plus<>;
+BENCHMARK_BINARY(add);
+
+using sub_op = cuda::std::minus<>;
+BENCHMARK_BINARY(sub);
+
+using mul_op = cuda::std::multiplies<>;
+BENCHMARK_BINARY(mul);
+
+using div_op = cuda::std::divides<>;
+BENCHMARK_BINARY(div);
+
+using le_op = cuda::std::less_equal<>;
+BENCHMARK_BINARY(le);
+
+using ge_op = cuda::std::greater_equal<>;
+BENCHMARK_BINARY(ge);
+
+struct fmin_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T a, T b) const
+  {
+    return ::cuda::std::fmin(a, b);
+  }
+};
+BENCHMARK_BINARY(fmin);
+
+struct fmax_op
+{
+  template <typename T>
+  _CCCL_HOST_DEVICE_API auto operator()(T a, T b) const
+  {
+    return ::cuda::std::fmax(a, b);
+  }
+};
+BENCHMARK_BINARY(fmax);

cccl_upstream/cub/benchmarks/bench/transform/tile/babelstream.cu

@@ -0,0 +1,216 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Tile variant of the BabelStream transform bench. The lambdas of the base benchmark are replaced by
+// named, stateless ops that register a tile_operator substitute (gated). Under --enable-tile +
+// CCCL_ENABLE_EXPERIMENTAL_TILE_TRANSFORM_DISPATCH the dispatch hook routes them to the tile kernel; otherwise this
+// is the standard CUB transform path. This file disappears once tile dispatch is fully transparent.
+
+#include "../common.h"
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+#  include <cuda_tile.h>
+#endif
+
+// Stateless scalar ops, used at the call site in both build modes. Constants are baked in so the ops
+// stay stateless (the tile substitute must be trivially default constructible): with startScalar == -2,
+// `c * scalar` is `-(c + c)`, `b + scalar * c` is `b - c - c`, etc.
+struct mul_op
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class B>
+  _CCCL_API auto operator()(B b) const
+  {
+    return -(b + b);
+  }
+};
+struct add_op
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class A, class B>
+  _CCCL_API auto operator()(A a, B b) const
+  {
+    return a + b;
+  }
+};
+struct triad_op
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class B, class C>
+  _CCCL_API auto operator()(B b, C c) const
+  {
+    return b - c - c;
+  }
+};
+struct nstream_op
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class A, class B, class C>
+  _CCCL_API auto operator()(A a, B b, C c) const
+  {
+    return a + b - c - c;
+  }
+};
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+CUB_NAMESPACE_BEGIN
+namespace detail::transform::tile
+{
+template <class T>
+inline constexpr bool tile_eligible_v<mul_op, T, 1> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<add_op, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<triad_op, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<nstream_op, T, 3> = true;
+template <>
+struct tile_operator<mul_op>
+{
+  using type = mul_op;
+};
+template <>
+struct tile_operator<add_op>
+{
+  using type = add_op;
+};
+template <>
+struct tile_operator<triad_op>
+{
+  using type = triad_op;
+};
+template <>
+struct tile_operator<nstream_op>
+{
+  using type = nstream_op;
+};
+} // namespace detail::transform::tile
+CUB_NAMESPACE_END
+#endif // _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+
+// The tile path does not support __int128 (no tensor_span/partition_view for it), so the type axis
+// omits it relative to the base babelstream bench.
+#ifdef TUNE_T
+using element_types = nvbench::type_list<TUNE_T>;
+#else
+using element_types = nvbench::type_list<nvbench::int8_t, nvbench::int16_t, nvbench::float32_t, nvbench::float64_t>;
+#endif
+
+inline auto array_size_powers = nvbench::range(16, 32, 4);
+
+// Same constant inputs as the base bench so nstream maintains a consistent workload.
+inline constexpr auto startA      = 11;
+inline constexpr auto startB      = 2;
+inline constexpr auto startC      = 1;
+inline constexpr auto startScalar = -2;
+static_assert(startA == (startA + startB + startScalar * startC), "nstream must have a consistent workload");
+
+template <typename T>
+static void mul(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(state, cuda::std::tuple{c.begin() + unaligned}, b.begin() + unaligned, n, mul_op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(mul, NVBENCH_TYPE_AXES(element_types))
+  .set_name("tile_mul")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);
+
+template <typename T>
+static void add(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+  thrust::device_vector<T> a(n + unaligned, startA);
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(
+    state, cuda::std::tuple{a.begin() + unaligned, b.begin() + unaligned}, c.begin() + unaligned, n, add_op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(add, NVBENCH_TYPE_AXES(element_types))
+  .set_name("tile_add")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);
+
+template <typename T>
+static void triad(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+  thrust::device_vector<T> a(n + unaligned, startA);
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(
+    state, cuda::std::tuple{b.begin() + unaligned, c.begin() + unaligned}, a.begin() + unaligned, n, triad_op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(triad, NVBENCH_TYPE_AXES(element_types))
+  .set_name("tile_triad")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);
+
+template <typename T>
+static void nstream(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n         = state.get_int64("Elements{io}");
+  const bool unaligned = state.get_string("Aligned") == "no";
+  thrust::device_vector<T> a(n + unaligned, startA);
+  thrust::device_vector<T> b(n + unaligned, startB);
+  thrust::device_vector<T> c(n + unaligned, startC);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(3 * n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(
+    state,
+    cuda::std::tuple{a.begin() + unaligned, b.begin() + unaligned, c.begin() + unaligned},
+    a.begin() + unaligned,
+    n,
+    nstream_op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(nstream, NVBENCH_TYPE_AXES(element_types))
+  .set_name("tile_nstream")
+  .set_type_axes_names({"T{ct}"})
+  .add_string_axis("Aligned", {"yes", "no"})
+  .add_int64_power_of_two_axis("Elements{io}", array_size_powers);

cccl_upstream/cub/benchmarks/bench/transform/tile/copy.cu

@@ -0,0 +1,69 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Pure copy (identity transform) -- measures plain load/store bandwidth through the tile
+// load_masked/store_masked path. The identity op registers a tile_operator substitute (gated); under
+// --enable-tile + CCCL_ENABLE_EXPERIMENTAL_TILE_TRANSFORM_DISPATCH the dispatch hook routes it to the tile kernel,
+// otherwise it falls through to CUB's standard transform. This file disappears once tile dispatch is
+// fully transparent.
+
+#include "../common.h"
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+#  include <cuda_tile.h>
+#endif
+
+struct identity
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class T>
+  _CCCL_API auto operator()(T v) const
+  {
+    return v;
+  }
+};
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+CUB_NAMESPACE_BEGIN
+namespace detail::transform::tile
+{
+template <class T>
+inline constexpr bool tile_eligible_v<identity, T, 1> = true;
+template <>
+struct tile_operator<identity>
+{
+  using type = identity;
+};
+} // namespace detail::transform::tile
+CUB_NAMESPACE_END
+#endif // _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+
+#ifdef TUNE_T
+using element_types = nvbench::type_list<TUNE_T>;
+#else
+using element_types = nvbench::type_list<nvbench::int8_t, nvbench::int16_t, nvbench::int32_t, nvbench::float64_t>;
+#endif
+
+template <typename T>
+static void copy(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+
+  thrust::device_vector<T> in = generate(n);
+  thrust::device_vector<T> out(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(state, cuda::std::tuple{in.begin()}, out.begin(), n, identity{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(copy, NVBENCH_TYPE_AXES(element_types))
+  .set_name("tile_copy")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/tile/grayscale.cu

@@ -0,0 +1,73 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Tile variant of the grayscale transform bench. Unlike the base bench (a single rgb_t<T> struct
+// input), this uses three separate R/G/B streams so the inputs are plain element types the tile path
+// can vectorize. The named rgb_to_y op registers a tile_operator substitute (gated). This file
+// disappears once tile dispatch is fully transparent.
+
+#include "../common.h"
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+#  include <cuda_tile.h>
+#endif
+
+struct rgb_to_y
+{
+  _CCCL_EXEC_CHECK_DISABLE
+  template <class R, class G, class B>
+  _CCCL_API auto operator()(R r, G g, B b) const
+  {
+    constexpr float w_r = 0.2989f;
+    constexpr float w_g = 0.587f;
+    constexpr float w_b = 0.114f;
+    return w_r * r + w_g * g + w_b * b;
+  }
+};
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+CUB_NAMESPACE_BEGIN
+namespace detail::transform::tile
+{
+template <class T>
+inline constexpr bool tile_eligible_v<rgb_to_y, T, 3> = true;
+template <>
+struct tile_operator<rgb_to_y>
+{
+  using type = rgb_to_y;
+};
+} // namespace detail::transform::tile
+CUB_NAMESPACE_END
+#endif // _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+
+#ifdef TUNE_T
+using value_types = nvbench::type_list<TUNE_T>;
+#else
+using value_types = nvbench::type_list<nvbench::float32_t, nvbench::float64_t>;
+#endif
+
+template <typename T>
+static void grayscale(nvbench::state& state, nvbench::type_list<T>)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+
+  thrust::device_vector<T> r = generate(n);
+  thrust::device_vector<T> g = generate(n);
+  thrust::device_vector<T> b = generate(n);
+  thrust::device_vector<T> out(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(3 * n); // matches the base bench's rgb_t<T> = 3 * sizeof(T)
+  state.add_global_memory_writes<T>(n);
+  bench_transform(state, cuda::std::tuple{r.begin(), g.begin(), b.begin()}, out.begin(), n, rgb_to_y{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+NVBENCH_BENCH_TYPES(grayscale, NVBENCH_TYPE_AXES(value_types))
+  .set_name("tile_grayscale")
+  .set_type_axes_names({"T{ct}"})
+  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(16, 32, 4));

cccl_upstream/cub/benchmarks/bench/transform/tile/pytorch.cu

@@ -0,0 +1,493 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+// Tile variant of the PyTorch-style transform benches. Each named op registers a tile_operator
+// substitute (gated); MUFU-heavy ops also opt into tile_mufu_heavy_v so the tile policy picker caps
+// items/thread at the vector width on sub-4-byte types. Under --enable-tile +
+// CCCL_ENABLE_EXPERIMENTAL_TILE_TRANSFORM_DISPATCH the dispatch hook routes them to the tile kernel; otherwise this
+// is the standard CUB path. This file disappears once tile dispatch is fully transparent.
+
+#include <cuda_bf16.h>
+#include <cuda_fp16.h>
+
+#include <cuda/std/cmath>
+
+#include "../common.h"
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+#  include <cuda_tile.h>
+#endif
+
+// Scalar ops the user passes to Transform. Sub-4-byte input types compute in float and cast back,
+// matching the tile substitutes below.
+template <class T>
+__host__ __device__ float to_f(T v)
+{
+  return static_cast<float>(v);
+}
+template <class T>
+__host__ __device__ T from_f(float f)
+{
+  return static_cast<T>(f);
+}
+
+struct relu_op
+{
+  template <class T>
+  __host__ __device__ T operator()(T v) const
+  {
+    float f = to_f(v);
+    return from_f<T>(f > 0.0f ? f : 0.0f);
+  }
+};
+struct sigmoid_op
+{
+  template <class T>
+  __host__ __device__ T operator()(T v) const
+  {
+    float f = to_f(v);
+    return from_f<T>(1.0f / (1.0f + ::cuda::std::exp(-f)));
+  }
+};
+struct tanh_op
+{
+  template <class T>
+  __host__ __device__ T operator()(T v) const
+  {
+    return from_f<T>(::cuda::std::tanh(to_f(v)));
+  }
+};
+struct gelu_op
+{
+  template <class T>
+  __host__ __device__ T operator()(T v) const
+  {
+    constexpr float k0 = 0.7978845608028654f, k1 = 0.044715f;
+    float f = to_f(v);
+    return from_f<T>(0.5f * f * (1.0f + ::cuda::std::tanh(k0 * (f + k1 * f * f * f))));
+  }
+};
+struct sin_op
+{
+  template <class T>
+  __host__ __device__ T operator()(T v) const
+  {
+    return from_f<T>(::cuda::std::sin(to_f(v)));
+  }
+};
+struct exp_op
+{
+  template <class T>
+  __host__ __device__ T operator()(T v) const
+  {
+    return from_f<T>(::cuda::std::exp(to_f(v)));
+  }
+};
+
+struct binary_add
+{
+  template <class A, class B>
+  __host__ __device__ auto operator()(A a, B b) const
+  {
+    return a + b;
+  }
+};
+struct binary_sub
+{
+  template <class A, class B>
+  __host__ __device__ auto operator()(A a, B b) const
+  {
+    return a - b;
+  }
+};
+struct binary_mul
+{
+  template <class A, class B>
+  __host__ __device__ auto operator()(A a, B b) const
+  {
+    return a * b;
+  }
+};
+struct binary_div
+{
+  template <class A, class B>
+  __host__ __device__ auto operator()(A a, B b) const
+  {
+    return a / b;
+  }
+};
+struct binary_le
+{
+  template <class A, class B>
+  __host__ __device__ A operator()(A a, B b) const
+  {
+    return static_cast<A>(a <= b);
+  }
+};
+struct binary_ge
+{
+  template <class A, class B>
+  __host__ __device__ A operator()(A a, B b) const
+  {
+    return static_cast<A>(a >= b);
+  }
+};
+struct binary_fmin
+{
+  template <class A, class B>
+  __host__ __device__ auto operator()(A a, B b) const
+  {
+    return a < b ? a : b;
+  }
+};
+struct binary_fmax
+{
+  template <class A, class B>
+  __host__ __device__ auto operator()(A a, B b) const
+  {
+    return a > b ? a : b;
+  }
+};
+
+#if _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+namespace ct = ::cuda::tiles;
+
+template <class T>
+__tile__ auto as_float(T v)
+{
+  return ct::element_cast<float>(v);
+}
+template <class T, class F>
+__tile__ auto from_float(F f)
+{
+  return ct::element_cast<ct::tile_element_t<T>>(f);
+}
+
+struct tile_relu
+{
+  template <class T>
+  __tile__ auto operator()(T v) const
+  {
+    auto f = as_float(v);
+    return from_float<T>(ct::select(f > 0.0f, f, f - f));
+  }
+};
+struct tile_sigmoid
+{
+  template <class T>
+  __tile__ auto operator()(T v) const
+  {
+    auto f = as_float(v);
+    return from_float<T>(1.0f / (1.0f + ct::exp(-f)));
+  }
+};
+struct tile_tanh
+{
+  template <class T>
+  __tile__ auto operator()(T v) const
+  {
+    return from_float<T>(ct::tanh(as_float(v)));
+  }
+};
+struct tile_gelu
+{
+  template <class T>
+  __tile__ auto operator()(T v) const
+  {
+    constexpr float k0 = 0.7978845608028654f, k1 = 0.044715f;
+    auto f = as_float(v);
+    return from_float<T>(0.5f * f * (1.0f + ct::tanh(k0 * (f + k1 * f * f * f))));
+  }
+};
+struct tile_sin
+{
+  template <class T>
+  __tile__ auto operator()(T v) const
+  {
+    return from_float<T>(ct::sin(as_float(v)));
+  }
+};
+struct tile_exp
+{
+  template <class T>
+  __tile__ auto operator()(T v) const
+  {
+    return from_float<T>(ct::exp(as_float(v)));
+  }
+};
+
+struct tile_binary_add
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return a + b;
+  }
+};
+struct tile_binary_sub
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return a - b;
+  }
+};
+struct tile_binary_mul
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return a * b;
+  }
+};
+struct tile_binary_div
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return a / b;
+  }
+};
+struct tile_binary_le
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return ct::element_cast<ct::tile_element_t<A>>(a <= b);
+  }
+};
+struct tile_binary_ge
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return ct::element_cast<ct::tile_element_t<A>>(a >= b);
+  }
+};
+struct tile_binary_fmin
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return ct::select(a < b, a, b);
+  }
+};
+struct tile_binary_fmax
+{
+  template <class A, class B>
+  __tile__ auto operator()(A a, B b) const
+  {
+    return ct::select(a > b, a, b);
+  }
+};
+
+CUB_NAMESPACE_BEGIN
+namespace detail::transform::tile
+{
+// Unary
+template <class T>
+inline constexpr bool tile_eligible_v<relu_op, T, 1> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<sigmoid_op, T, 1> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<tanh_op, T, 1> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<gelu_op, T, 1> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<sin_op, T, 1> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<exp_op, T, 1> = true;
+template <>
+struct tile_operator<relu_op>
+{
+  using type = tile_relu;
+};
+template <>
+struct tile_operator<sigmoid_op>
+{
+  using type = tile_sigmoid;
+};
+template <>
+struct tile_operator<tanh_op>
+{
+  using type = tile_tanh;
+};
+template <>
+struct tile_operator<gelu_op>
+{
+  using type = tile_gelu;
+};
+template <>
+struct tile_operator<sin_op>
+{
+  using type = tile_sin;
+};
+template <>
+struct tile_operator<exp_op>
+{
+  using type = tile_exp;
+};
+
+// MUFU-heavy unary ops: hint the tile policy picker to cap items/thread at the vector width on
+// sub-4-byte types.
+template <>
+inline constexpr bool tile_mufu_heavy_v<sigmoid_op> = true;
+template <>
+inline constexpr bool tile_mufu_heavy_v<tanh_op> = true;
+template <>
+inline constexpr bool tile_mufu_heavy_v<gelu_op> = true;
+template <>
+inline constexpr bool tile_mufu_heavy_v<sin_op> = true;
+template <>
+inline constexpr bool tile_mufu_heavy_v<exp_op> = true;
+
+// Binary
+template <class T>
+inline constexpr bool tile_eligible_v<binary_add, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_sub, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_mul, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_div, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_le, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_ge, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_fmin, T, 2> = true;
+template <class T>
+inline constexpr bool tile_eligible_v<binary_fmax, T, 2> = true;
+template <>
+struct tile_operator<binary_add>
+{
+  using type = tile_binary_add;
+};
+template <>
+struct tile_operator<binary_sub>
+{
+  using type = tile_binary_sub;
+};
+template <>
+struct tile_operator<binary_mul>
+{
+  using type = tile_binary_mul;
+};
+template <>
+struct tile_operator<binary_div>
+{
+  using type = tile_binary_div;
+};
+template <>
+struct tile_operator<binary_le>
+{
+  using type = tile_binary_le;
+};
+template <>
+struct tile_operator<binary_ge>
+{
+  using type = tile_binary_ge;
+};
+template <>
+struct tile_operator<binary_fmin>
+{
+  using type = tile_binary_fmin;
+};
+template <>
+struct tile_operator<binary_fmax>
+{
+  using type = tile_binary_fmax;
+};
+} // namespace detail::transform::tile
+CUB_NAMESPACE_END
+#endif // _CCCL_CUB_TILE_TRANSFORM_DISPATCH_ENABLED()
+
+#ifdef TUNE_T
+using element_types = nvbench::type_list<TUNE_T>;
+#else
+using element_types = nvbench::type_list<
+#  if _CCCL_HAS_NVFP16() && _CCCL_CTK_AT_LEAST(12, 2)
+  __half,
+#  endif
+#  if _CCCL_HAS_NVBF16() && _CCCL_CTK_AT_LEAST(12, 2)
+  __nv_bfloat16,
+#  endif
+  nvbench::float32_t>;
+#endif
+
+template <typename Op, typename T>
+static void run_unary(nvbench::state& state)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+  thrust::device_vector<T> in(n, T(1));
+  thrust::device_vector<T> out(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(state, cuda::std::tuple{in.begin()}, out.begin(), n, Op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+template <typename Op, typename T>
+static void run_binary(nvbench::state& state)
+try
+{
+  const auto n = state.get_int64("Elements{io}");
+  thrust::device_vector<T> a(n, T(1));
+  thrust::device_vector<T> b(n, T(1));
+  thrust::device_vector<T> out(n, thrust::no_init);
+
+  state.add_element_count(n);
+  state.add_global_memory_reads<T>(2 * n);
+  state.add_global_memory_writes<T>(n);
+  bench_transform(state, cuda::std::tuple{a.begin(), b.begin()}, out.begin(), n, Op{});
+}
+catch (const std::bad_alloc&)
+{
+  state.skip("Skipping: out of memory.");
+}
+
+inline auto pt_sizes = nvbench::range(16, 32, 4);
+
+#define UNARY_BENCH(name, op)                                            \
+  template <typename T>                                                  \
+  static void name##_bench(nvbench::state& state, nvbench::type_list<T>) \
+  {                                                                      \
+    run_unary<op, T>(state);                                             \
+  }                                                                      \
+  NVBENCH_BENCH_TYPES(name##_bench, NVBENCH_TYPE_AXES(element_types))    \
+    .set_name("tile_" #name)                                             \
+    .set_type_axes_names({"T{ct}"})                                      \
+    .add_int64_power_of_two_axis("Elements{io}", pt_sizes)
+
+UNARY_BENCH(relu, relu_op);
+UNARY_BENCH(sigmoid, sigmoid_op);
+UNARY_BENCH(tanh, tanh_op);
+UNARY_BENCH(gelu, gelu_op);
+UNARY_BENCH(sin, sin_op);
+UNARY_BENCH(exp, exp_op);
+
+#define BINARY_BENCH(name, op)                                           \
+  template <typename T>                                                  \
+  static void name##_bench(nvbench::state& state, nvbench::type_list<T>) \
+  {                                                                      \
+    run_binary<op, T>(state);                                            \
+  }                                                                      \
+  NVBENCH_BENCH_TYPES(name##_bench, NVBENCH_TYPE_AXES(element_types))    \
+    .set_name("tile_pt_" #name)                                          \
+    .set_type_axes_names({"T{ct}"})                                      \
+    .add_int64_power_of_two_axis("Elements{io}", pt_sizes)
+
+BINARY_BENCH(add, binary_add);
+BINARY_BENCH(sub, binary_sub);
+BINARY_BENCH(mul, binary_mul);
+BINARY_BENCH(div, binary_div);
+BINARY_BENCH(le, binary_le);
+BINARY_BENCH(ge, binary_ge);
+BINARY_BENCH(fmin, binary_fmin);
+BINARY_BENCH(fmax, binary_fmax);