[INFRA] Import NVIDIA/CCCL upstream as optimization reference library
CCCL (CUDA C++ Core Libraries) provides: - CUB: device/block/warp-level GPU primitives (reduce, scan, sort, topk) - Thrust: high-level parallel algorithms (transform_reduce, sort, scan) - libcudacxx: CUDA C++ standard library (atomics, barriers, memory) - cudax: experimental features (memory resources, allocators) - Tuning policies: per-SM hardware-specific algorithm parameters Competition optimization vectors mapped to CCCL: - Output TPS (83% weight): warp_reduce, block_reduce, device_topk - Input TPS (14% weight): device_scan, block_load, prefetch - Cache TPS (3% weight): prefix caching strategy patterns - Memory (0.9 util): pooled/cached/buddy allocators Source: https://github.com/NVIDIA/cccl (shallow clone, HEAD only) License: Apache-2.0
This commit is contained in:
153
cccl_upstream/cub/benchmarks/bench/segmented_scan/base.cuh
Normal file
153
cccl_upstream/cub/benchmarks/bench/segmented_scan/base.cuh
Normal file
@@ -0,0 +1,153 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cub/device/device_segmented_scan.cuh>
|
||||
|
||||
#include <thrust/tabulate.h>
|
||||
|
||||
#include <cuda/std/__functional/invoke.h>
|
||||
#include <cuda/std/type_traits>
|
||||
|
||||
#include <nvbench_helper.cuh>
|
||||
|
||||
#if !TUNE_BASE
|
||||
# if TUNE_TRANSPOSE == 0
|
||||
# define TUNE_BLOCK_LOAD_ALGORITHM cub::BLOCK_LOAD_DIRECT
|
||||
# define TUNE_BLOCK_STORE_ALGORITHM cub::BLOCK_STORE_DIRECT
|
||||
# else // TUNE_TRANSPOSE == 1
|
||||
# define TUNE_BLOCK_LOAD_ALGORITHM cub::BLOCK_LOAD_WARP_TRANSPOSE
|
||||
# define TUNE_BLOCK_STORE_ALGORITHM cub::BLOCK_STORE_WARP_TRANSPOSE
|
||||
# endif // TUNE_TRANSPOSE
|
||||
|
||||
# if TUNE_LOAD == 0
|
||||
# define TUNE_LOAD_MODIFIER cub::LOAD_DEFAULT
|
||||
# elif TUNE_LOAD == 1
|
||||
# define TUNE_LOAD_MODIFIER cub::LOAD_CA
|
||||
# endif // TUNE_LOAD
|
||||
|
||||
template <int ThreadsPerBlock, int ItemsPerThread, int MaxSegmentsPerBlock>
|
||||
struct policy_selector_t
|
||||
{
|
||||
[[nodiscard]] _CCCL_HOST_DEVICE constexpr auto operator()(cuda::compute_capability) const -> cub::SegmentedScanPolicy
|
||||
{
|
||||
return cub::SegmentedScanPolicy{cub::SegmentedScanBlockPolicy{
|
||||
ThreadsPerBlock,
|
||||
ItemsPerThread,
|
||||
TUNE_BLOCK_LOAD_ALGORITHM,
|
||||
TUNE_LOAD_MODIFIER,
|
||||
TUNE_BLOCK_STORE_ALGORITHM,
|
||||
cub::BLOCK_SCAN_WARP_SCANS,
|
||||
MaxSegmentsPerBlock}};
|
||||
}
|
||||
};
|
||||
#endif // TUNE_BASE
|
||||
|
||||
template <typename OffsetT>
|
||||
struct to_offsets_functor
|
||||
{
|
||||
OffsetT elements;
|
||||
OffsetT segment_size;
|
||||
OffsetT wobble;
|
||||
|
||||
__host__ __device__ __forceinline__ OffsetT operator()(size_t i) const
|
||||
{
|
||||
const auto fixed_size_value = static_cast<OffsetT>(i) * segment_size;
|
||||
const auto correction = ((i & 1) ? wobble : OffsetT{0});
|
||||
|
||||
return cuda::std::min(elements, fixed_size_value + correction);
|
||||
}
|
||||
};
|
||||
|
||||
template <size_t Wobble = 0, typename T, typename OffsetT>
|
||||
static void bench_impl(nvbench::state& state, nvbench::type_list<T, OffsetT>)
|
||||
{
|
||||
#if !TUNE_BASE
|
||||
using policy_t = policy_selector_t<TUNE_THREADS, TUNE_ITEMS, TUNE_MAX_SEGMENTS_PER_BLOCK>;
|
||||
#endif
|
||||
|
||||
const auto elements = static_cast<OffsetT>(state.get_int64("Elements{io}"));
|
||||
const auto segment_size = static_cast<OffsetT>(state.get_int64("SegmentSize{io}"));
|
||||
const auto num_segments = cuda::ceil_div(elements, segment_size);
|
||||
auto& summary = state.add_summary("user/derived/segment_count");
|
||||
summary.set_string("name", "#Segments");
|
||||
summary.set_int64("value", num_segments);
|
||||
|
||||
thrust::device_vector<T> input = generate(elements);
|
||||
thrust::device_vector<T> output(elements, thrust::default_init);
|
||||
|
||||
thrust::device_vector<OffsetT> offsets(num_segments + 1, thrust::no_init);
|
||||
thrust::tabulate(offsets.begin(), offsets.end(), to_offsets_functor<OffsetT>{elements, segment_size, Wobble});
|
||||
|
||||
const T* d_input = thrust::raw_pointer_cast(input.data());
|
||||
T* d_output = thrust::raw_pointer_cast(output.data());
|
||||
const OffsetT* d_offsets = thrust::raw_pointer_cast(offsets.data());
|
||||
|
||||
state.add_element_count(elements, "Elements");
|
||||
state.add_global_memory_reads<T>(elements);
|
||||
state.add_global_memory_reads<OffsetT>(num_segments + 1);
|
||||
state.add_global_memory_writes<T>(elements);
|
||||
|
||||
caching_allocator_t alloc;
|
||||
state.exec(nvbench::exec_tag::gpu | nvbench::exec_tag::no_batch, [&](nvbench::launch& launch) {
|
||||
auto env = cub_bench_env(
|
||||
alloc,
|
||||
launch
|
||||
#if !TUNE_BASE
|
||||
,
|
||||
cuda::execution::tune(policy_t{})
|
||||
#endif // !TUNE_BASE
|
||||
);
|
||||
_CCCL_TRY_CUDA_API(
|
||||
cub::DeviceSegmentedScan::ExclusiveSegmentedScan,
|
||||
"ExclusiveSegmentedScan failed",
|
||||
d_input,
|
||||
d_output,
|
||||
d_offsets,
|
||||
d_offsets + 1,
|
||||
d_offsets,
|
||||
num_segments,
|
||||
op_t{},
|
||||
T{},
|
||||
env);
|
||||
});
|
||||
}
|
||||
|
||||
template <typename T, typename OffsetT>
|
||||
static void fixed_segment_size_bench(nvbench::state& state, nvbench::type_list<T, OffsetT> tl)
|
||||
{
|
||||
return bench_impl<0, T, OffsetT>(state, tl);
|
||||
}
|
||||
|
||||
template <typename T, typename OffsetT>
|
||||
static void varying_segment_size_bench(nvbench::state& state, nvbench::type_list<T, OffsetT> tl)
|
||||
{
|
||||
return bench_impl<1, T, OffsetT>(state, tl);
|
||||
}
|
||||
|
||||
#if (_CCCL_CUDA_COMPILER(NVCC, >=, 12, 1))
|
||||
using benched_value_types = all_types;
|
||||
#else
|
||||
// WAR for excessive time CTK 12.0 CICC takes to compile these benchmarks for int128_t
|
||||
# ifdef TUNE_T
|
||||
static_assert(!cuda::std::is_integral_v<TUNE_T> || sizeof(TUNE_T) < 16);
|
||||
using benched_value_types = nvbench::type_list<TUNE_T>;
|
||||
# else
|
||||
using benched_value_types = nvbench::type_list<int8_t, int16_t, int32_t, int64_t, float, double, complex32>;
|
||||
# endif
|
||||
#endif
|
||||
|
||||
NVBENCH_BENCH_TYPES(fixed_segment_size_bench, NVBENCH_TYPE_AXES(benched_value_types, offset_types))
|
||||
.set_name("fixed_size_segments")
|
||||
.set_type_axes_names({"T{ct}", "OffsetT{ct}"})
|
||||
.add_int64_power_of_two_axis("Elements{io}", nvbench::range(18, 26, 4))
|
||||
.add_int64_axis("SegmentSize{io}", {51, 123, 233, 513, 1337, 4417});
|
||||
|
||||
NVBENCH_BENCH_TYPES(varying_segment_size_bench, NVBENCH_TYPE_AXES(benched_value_types, offset_types))
|
||||
.set_name("varying_size_segments")
|
||||
.set_type_axes_names({"T{ct}", "OffsetT{ct}"})
|
||||
.add_int64_power_of_two_axis("Elements{io}", nvbench::range(18, 26, 4))
|
||||
.add_int64_axis("SegmentSize{io}", {51, 123, 233, 513, 1337, 4417});
|
||||
// .add_int64_axis("SegmentsPerWorker{io}", {1}) // public API doesn' expose them (yet)
|
||||
// .add_string_axis("Worker{io}", {"block"});
|
||||
Reference in New Issue
Block a user