project_6/cccl_upstream/cub/benchmarks/bench/segmented_scan/base.cuh

// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

#pragma once

#include <cub/device/device_segmented_scan.cuh>

#include <thrust/tabulate.h>

#include <cuda/std/__functional/invoke.h>
#include <cuda/std/type_traits>

#include <nvbench_helper.cuh>

#if !TUNE_BASE
#  if TUNE_TRANSPOSE == 0
#    define TUNE_BLOCK_LOAD_ALGORITHM  cub::BLOCK_LOAD_DIRECT
#    define TUNE_BLOCK_STORE_ALGORITHM cub::BLOCK_STORE_DIRECT
#  else // TUNE_TRANSPOSE == 1
#    define TUNE_BLOCK_LOAD_ALGORITHM  cub::BLOCK_LOAD_WARP_TRANSPOSE
#    define TUNE_BLOCK_STORE_ALGORITHM cub::BLOCK_STORE_WARP_TRANSPOSE
#  endif // TUNE_TRANSPOSE

#  if TUNE_LOAD == 0
#    define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
#  elif TUNE_LOAD == 1
#    define TUNE_LOAD_MODIFIER cub::LOAD_CA
#  endif // TUNE_LOAD

template <int ThreadsPerBlock, int ItemsPerThread, int MaxSegmentsPerBlock>
struct policy_selector_t
{
  [[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::SegmentedScanPolicy
  {
    return cub::SegmentedScanPolicy{cub::SegmentedScanBlockPolicy{
      ThreadsPerBlock,
      ItemsPerThread,
      TUNE_BLOCK_LOAD_ALGORITHM,
      TUNE_LOAD_MODIFIER,
      TUNE_BLOCK_STORE_ALGORITHM,
      cub::BLOCK_SCAN_WARP_SCANS,
      MaxSegmentsPerBlock}};
  }
};
#endif // TUNE_BASE

template <typename OffsetT>
struct to_offsets_functor
{
  OffsetT elements;
  OffsetT segment_size;
  OffsetT wobble;

  __host__ __device__ __forceinline__ OffsetT operator()(size_t i) const
  {
    const auto fixed_size_value = static_cast<OffsetT>(i) * segment_size;
    const auto correction       = ((i & 1) ? wobble : OffsetT{0});

    return cuda::std::min(elements, fixed_size_value + correction);
  }
};

template <size_t Wobble = 0, typename T, typename OffsetT>
static void bench_impl(nvbench::state& state, nvbench::type_list<T, OffsetT>)
{
#if !TUNE_BASE
  using policy_t = policy_selector_t<TUNE_THREADS, TUNE_ITEMS, TUNE_MAX_SEGMENTS_PER_BLOCK>;
#endif

  const auto elements     = static_cast<OffsetT>(state.get_int64("Elements{io}"));
  const auto segment_size = static_cast<OffsetT>(state.get_int64("SegmentSize{io}"));
  const auto num_segments = cuda::ceil_div(elements, segment_size);
  auto& summary           = state.add_summary("user/derived/segment_count");
  summary.set_string("name", "#Segments");
  summary.set_int64("value", num_segments);

  thrust::device_vector<T> input = generate(elements);
  thrust::device_vector<T> output(elements, thrust::default_init);

  thrust::device_vector<OffsetT> offsets(num_segments + 1, thrust::no_init);
  thrust::tabulate(offsets.begin(), offsets.end(), to_offsets_functor<OffsetT>{elements, segment_size, Wobble});

  const T* d_input         = thrust::raw_pointer_cast(input.data());
  T* d_output              = thrust::raw_pointer_cast(output.data());
  const OffsetT* d_offsets = thrust::raw_pointer_cast(offsets.data());

  state.add_element_count(elements, "Elements");
  state.add_global_memory_reads<T>(elements);
  state.add_global_memory_reads<OffsetT>(num_segments + 1);
  state.add_global_memory_writes<T>(elements);

  caching_allocator_t alloc;
  state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
    auto env = cub_bench_env(
      alloc,
      launch
#if !TUNE_BASE
      ,
      cuda::execution::tune(policy_t{})
#endif // !TUNE_BASE
    );
    _CCCL_TRY_CUDA_API(
      cub::DeviceSegmentedScan::ExclusiveSegmentedScan,
      "ExclusiveSegmentedScan failed",
      d_input,
      d_output,
      d_offsets,
      d_offsets + 1,
      d_offsets,
      num_segments,
      op_t{},
      T{},
      env);
  });
}

template <typename T, typename OffsetT>
static void fixed_segment_size_bench(nvbench::state& state, nvbench::type_list<T, OffsetT> tl)
{
  return bench_impl<0, T, OffsetT>(state, tl);
}

template <typename T, typename OffsetT>
static void varying_segment_size_bench(nvbench::state& state, nvbench::type_list<T, OffsetT> tl)
{
  return bench_impl<1, T, OffsetT>(state, tl);
}

#if (_CCCL_CUDA_COMPILER(NVCC, >=, 12, 1))
using benched_value_types = all_types;
#else
// WAR for excessive time CTK 12.0 CICC takes to compile these benchmarks for int128_t
#  ifdef TUNE_T
static_assert(!cuda::std::is_integral_v<TUNE_T> || sizeof(TUNE_T) < 16);
using benched_value_types = nvbench::type_list<TUNE_T>;
#  else
using benched_value_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double, complex32>;
#  endif
#endif

NVBENCH_BENCH_TYPES(fixed_segment_size_bench, NVBENCH_TYPE_AXES(benched_value_types, offset_types))
  .set_name("fixed_size_segments")
  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(18, 26, 4))
  .add_int64_axis("SegmentSize{io}", {51, 123, 233, 513, 1337, 4417});

NVBENCH_BENCH_TYPES(varying_segment_size_bench, NVBENCH_TYPE_AXES(benched_value_types, offset_types))
  .set_name("varying_size_segments")
  .set_type_axes_names({"T{ct}", "OffsetT{ct}"})
  .add_int64_power_of_two_axis("Elements{io}", nvbench::range(18, 26, 4))
  .add_int64_axis("SegmentSize{io}", {51, 123, 233, 513, 1337, 4417});
// .add_int64_axis("SegmentsPerWorker{io}", {1}) // public API doesn' expose them (yet)
// .add_string_axis("Worker{io}", {"block"});