[CCCL] 瘦身 + 补全: 移除 cudax/python/libcudacxx-tests 冗余文件, 新增 c2h 测试助手 + cmake 构建系统 + 8 个 CUDA thrust examples
变更摘要:
- 删除: cudax/ (783 files, 7.2M) — 实验性组件，竞赛不需要
- 删除: python/ (226 files, 2.0M) — Python 绑定，竞赛不需要
- 删除: libcudacxx/{test,benchmarks,codegen,cmake,share} (4432 files, 31M)
保留: libcudacxx/include/ (1463 headers, cuda::std 编译依赖)
- 新增: c2h/ (27 files) — CUB Catch2 测试辅助头文件，编译 243 个测试必需
- 新增: cmake/ (29 files) — CCCL 原生 CMake 构建系统
- 新增: thrust/examples/cuda/ (7 files) + cpp_integration/ (1 file)
async_reduce, custom_temporary_allocation, explicit_cuda_stream,
global_device_vector, range_view, unwrap_pointer, wrap_pointer, device
结果: cccl_upstream 从 74M→35M (瘦身 53%), 核心内容 100% 保留:
27/27 tuning headers, 78 benchmarks, 243 tests,
60 thrust examples, 18 CUB examples, 全部编译头文件
This commit is contained in:
muh-bot
@@ -1,331 +0,0 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
|
||||
#include <thrust/device_vector.h>
|
||||
|
||||
#include <cuda/mdspan>
|
||||
#include <cuda/std/array>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/copy.cuh>
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
|
||||
#include <nvbench/nvbench.cuh>
|
||||
|
||||
// GCC -Warray-bounds false positive for high-rank (20+) __raw_tensor instantiations
|
||||
_CCCL_DIAG_SUPPRESS_GCC("-Warray-bounds")
|
||||
|
||||
template <size_t Rank, typename idx_t>
|
||||
size_t
|
||||
compute_alloc(size_t offset, const cuda::std::array<idx_t, Rank>& shape, const cuda::std::array<idx_t, Rank>& strides)
|
||||
{
|
||||
int64_t max_pos = static_cast<int64_t>(offset);
|
||||
for (size_t i = 0; i < Rank; ++i)
|
||||
{
|
||||
auto delta = static_cast<ptrdiff_t>(shape[i] - 1) * strides[i];
|
||||
if (delta > 0)
|
||||
{
|
||||
max_pos += delta;
|
||||
}
|
||||
}
|
||||
return max_pos + 1;
|
||||
}
|
||||
|
||||
template <typename data_t = int, typename idx_t = int, size_t Rank>
|
||||
void bench_copy(nvbench::state& state,
|
||||
size_t src_offset,
|
||||
const cuda::std::array<idx_t, Rank>& shape,
|
||||
const cuda::std::array<idx_t, Rank>& src_strides,
|
||||
size_t dst_offset,
|
||||
const cuda::std::array<idx_t, Rank>& dst_strides)
|
||||
{
|
||||
const auto src_alloc = compute_alloc(src_offset, shape, src_strides);
|
||||
const auto dst_alloc = compute_alloc(dst_offset, shape, dst_strides);
|
||||
|
||||
thrust::device_vector<data_t> d_src(src_alloc);
|
||||
thrust::device_vector<data_t> d_dst(dst_alloc);
|
||||
|
||||
size_t num_items = 1;
|
||||
for (size_t i = 0; i < Rank; ++i)
|
||||
{
|
||||
num_items *= shape[i];
|
||||
}
|
||||
state.add_element_count(num_items);
|
||||
state.add_global_memory_reads<data_t>(num_items);
|
||||
state.add_global_memory_writes<data_t>(num_items);
|
||||
|
||||
using extents_t = cuda::std::dextents<idx_t, Rank>;
|
||||
using strides_t = cuda::dstrides<idx_t, Rank>;
|
||||
using mapping_t = cuda::layout_stride_relaxed::mapping<extents_t>;
|
||||
|
||||
extents_t ext(shape);
|
||||
auto src_ptr = thrust::raw_pointer_cast(d_src.data()) + src_offset;
|
||||
auto dst_ptr = thrust::raw_pointer_cast(d_dst.data()) + dst_offset;
|
||||
mapping_t src_map(ext, strides_t(src_strides));
|
||||
mapping_t dst_map(ext, strides_t(dst_strides));
|
||||
|
||||
cuda::device_mdspan<data_t, extents_t, cuda::layout_stride_relaxed> src(src_ptr, src_map);
|
||||
cuda::device_mdspan<data_t, extents_t, cuda::layout_stride_relaxed> dst(dst_ptr, dst_map);
|
||||
|
||||
state.exec([&](nvbench::launch& launch) {
|
||||
cuda::stream_ref stream{launch.get_stream()};
|
||||
cuda::experimental::copy(src, dst, stream);
|
||||
});
|
||||
}
|
||||
|
||||
template <typename data_t = int, typename idx_t = int, size_t Rank>
|
||||
void bench_copy(nvbench::state& state,
|
||||
size_t offset,
|
||||
const cuda::std::array<idx_t, Rank>& shape,
|
||||
const cuda::std::array<idx_t, Rank>& strides)
|
||||
{
|
||||
bench_copy<data_t>(state, offset, shape, strides, offset, strides);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Memcpy benchmarks
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (25, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1)
|
||||
// dst: (25, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1)
|
||||
void memcpy_layout_0(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 5> shape{25, 70, 90, 80, 80};
|
||||
cuda::std::array<int, 5> strides{40320000, 576000, 6400, 80, 1};
|
||||
bench_copy(state, 0, shape, strides);
|
||||
}
|
||||
NVBENCH_BENCH(memcpy_layout_0).set_name("contiguous (5D, int, 4GB)");
|
||||
|
||||
// src: (25, 80, 70, 80, 90):(40320000, 1, 576000, 80, 6400)
|
||||
// dst: (25, 80, 70, 80, 90):(40320000, 1, 576000, 80, 6400)
|
||||
void memcpy_layout_1(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 5> shape{25, 80, 70, 80, 90};
|
||||
cuda::std::array<int, 5> strides{40320000, 1, 576000, 80, 6400};
|
||||
bench_copy(state, 0, shape, strides);
|
||||
}
|
||||
NVBENCH_BENCH(memcpy_layout_1).set_name("contiguous-perm (5D, int, 4GB)");
|
||||
|
||||
// src: (1, 25, 1, 80, 1, 70, 1, 80, 1, 90):(1, 40320000, 1, 1, 1, 576000, 1, 80, 1, 6400)
|
||||
// dst: (1, 25, 1, 80, 1, 70, 1, 80, 1, 90):(1, 40320000, 1, 1, 1, 576000, 1, 80, 1, 6400)
|
||||
void memcpy_layout_1b(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 10> shape{1, 25, 1, 80, 1, 70, 1, 80, 1, 90};
|
||||
cuda::std::array<int, 10> strides{1, 40320000, 1, 1, 1, 576000, 1, 80, 1, 6400};
|
||||
bench_copy(state, 0, shape, strides);
|
||||
}
|
||||
NVBENCH_BENCH(memcpy_layout_1b).set_name("contiguous-1-sized (10D, int, 4GB)");
|
||||
|
||||
// src: (25, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1)
|
||||
// dst: (25, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1)
|
||||
void memcpy_layout_2(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 5> shape{25, 70, 90, 80, 80};
|
||||
cuda::std::array<int, 5> strides{40320000, 576000, 6400, 80, 1};
|
||||
bench_copy(state, 1, shape, strides);
|
||||
}
|
||||
NVBENCH_BENCH(memcpy_layout_2).set_name("contiguous-not-aligned (5D, int, 4GB)");
|
||||
|
||||
// src: (100, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1)
|
||||
// dst: (100, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1)
|
||||
void memcpy_layout_3(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int64_t, 5> shape{100, 70, 90, 80, 80};
|
||||
cuda::std::array<int64_t, 5> strides{40320000, 576000, 6400, 80, 1};
|
||||
bench_copy<char, int64_t>(state, 0, shape, strides);
|
||||
}
|
||||
NVBENCH_BENCH(memcpy_layout_3).set_name("contiguous-small (5D, char, 4GB)");
|
||||
|
||||
// src: (100, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1)
|
||||
// dst: (100, 70, 90, 80, 80):(40320000, 576000, 6400, 80, 1)
|
||||
void memcpy_layout_4(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int64_t, 5> shape{100, 70, 90, 80, 80};
|
||||
cuda::std::array<int64_t, 5> strides{40320000, 576000, 6400, 80, 1};
|
||||
bench_copy<char, int64_t>(state, 1, shape, strides);
|
||||
}
|
||||
NVBENCH_BENCH(memcpy_layout_4).set_name("contiguous-small-not-aligned (5D, char, 4GB)");
|
||||
|
||||
// src: (25, 70, 90, 80, 80):(40320000, 576000, 6400, 80, -1), offset=80
|
||||
// dst: (25, 70, 90, 80, 80):(40320000, 576000, 6400, 80, -1), offset=80
|
||||
void memcpy_neg(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 5> shape{25, 70, 90, 80, 80};
|
||||
cuda::std::array<int, 5> strides{40320000, 576000, 6400, 80, -1};
|
||||
bench_copy(state, 80, shape, strides);
|
||||
}
|
||||
NVBENCH_BENCH(memcpy_neg).set_name("contiguous-negative-stride (5D, int, 4GB)");
|
||||
|
||||
// src: (134217600, 32):(128, 1), offset=32
|
||||
// dst: (134217600, 32):(128, 1), offset=32
|
||||
// Copies 4GB while allocating 16GB per tensor because of the padded outer stride.
|
||||
void vectorization(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int64_t, 2> shape{134217600, 32};
|
||||
cuda::std::array<int64_t, 2> strides{128, 1};
|
||||
bench_copy<char, int64_t>(state, 32, shape, strides);
|
||||
}
|
||||
NVBENCH_BENCH(vectorization).set_name("vectorization (2D, char, 4GB copy, 16GB alloc)");
|
||||
|
||||
// src: (32767, (128 * 1024) / sizeof(int)):(128 * 1024, 1)
|
||||
// dst: (32767, (128 * 1024) / sizeof(int)):(128 * 1024, 1)
|
||||
// Copies 4GB while allocating 16GB per tensor because each row is padded to 128K elements.
|
||||
void block_contiguous(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 2> shape{32767, (128 * 1024) / sizeof(int)};
|
||||
cuda::std::array<int, 2> strides{128 * 1024, 1};
|
||||
bench_copy(state, 0, shape, strides);
|
||||
}
|
||||
NVBENCH_BENCH(block_contiguous).set_name("block-contiguous (2D, int, 4GB copy, 16GB alloc)");
|
||||
// (non-vectorizable)
|
||||
|
||||
void several_dimensions(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 5> shape{64, 64, 64, 64, 64};
|
||||
cuda::std::array<int, 5> strides{17043520 + 1, 266304 + 1, 4160 + 1, 64 + 1, 1};
|
||||
bench_copy(state, 0, shape, strides);
|
||||
}
|
||||
NVBENCH_BENCH(several_dimensions).set_name("several_dimensions (5D, int, 4GB)");
|
||||
|
||||
void several_dimensions_non_square(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 5> shape{63, 65, 67, 69, 57};
|
||||
cuda::std::array<int, 5> strides{17433131, 268202, 4003, 58, 1};
|
||||
bench_copy(state, 0, shape, strides);
|
||||
}
|
||||
NVBENCH_BENCH(several_dimensions_non_square).set_name("several_dimensions_non_square (5D, int, 4GB)");
|
||||
|
||||
/***********************************************************************************************************************
|
||||
* Transpose benchmark
|
||||
**********************************************************************************************************************/
|
||||
|
||||
// src: (32768,32768):(1,32768)
|
||||
// dst: (32768,32768):(32768,1)
|
||||
void transpose_2D_col_row(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 2> shape{32768, 32768};
|
||||
cuda::std::array<int, 2> src_strides{1, 32768};
|
||||
cuda::std::array<int, 2> dst_strides{32768, 1};
|
||||
bench_copy(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_2D_col_row).set_name("transpose_2D_col_row (2D, int, 4GB)");
|
||||
|
||||
void transpose_2D_row_col(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 2> shape{32768, 32768};
|
||||
cuda::std::array<int, 2> src_strides{32768, 1};
|
||||
cuda::std::array<int, 2> dst_strides{1, 32768};
|
||||
bench_copy(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_2D_row_col).set_name("transpose_2D_row_col (2D, int, 4GB)");
|
||||
|
||||
void transpose_2D_char(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int64_t, 2> shape{65536, 65536};
|
||||
cuda::std::array<int64_t, 2> src_strides{1, 65536};
|
||||
cuda::std::array<int64_t, 2> dst_strides{65536, 1};
|
||||
bench_copy<char, int64_t>(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_2D_char).set_name("transpose_2D_char (2D, char, 4GB)");
|
||||
|
||||
void transpose_2D_short(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 2> shape{32760, 32768 * 2};
|
||||
cuda::std::array<int, 2> src_strides{1, 32760};
|
||||
cuda::std::array<int, 2> dst_strides{32768 * 2, 1};
|
||||
bench_copy<short>(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_2D_short).set_name("transpose_2D_short (2D, short, 4GB)");
|
||||
|
||||
void transpose_2D_double(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int64_t, 2> shape{32768, 16384};
|
||||
cuda::std::array<int64_t, 2> src_strides{1, 32768};
|
||||
cuda::std::array<int64_t, 2> dst_strides{16384, 1};
|
||||
bench_copy<double, int64_t>(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_2D_double).set_name("transpose_2D_double (2D, double, 4GB)");
|
||||
|
||||
void transpose_2D_odd_both(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 2> shape{32767, 32769};
|
||||
cuda::std::array<int, 2> src_strides{1, 32767};
|
||||
cuda::std::array<int, 2> dst_strides{32769, 1};
|
||||
bench_copy(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_2D_odd_both).set_name("transpose_2D_odd_both (2D, int, 4GB)");
|
||||
|
||||
void transpose_3D(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 3> shape{1024, 1024, 1024};
|
||||
cuda::std::array<int, 3> src_strides{1, 1024, 1024 * 1024};
|
||||
cuda::std::array<int, 3> dst_strides{1024 * 1024, 1024, 1};
|
||||
bench_copy(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_3D).set_name("transpose_3D (3D, int, 4GB)");
|
||||
|
||||
void transpose_3D_odd_edges(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 3> shape{1023, 1025, 1024};
|
||||
cuda::std::array<int, 3> src_strides{1, 1023, 1023 * 1025};
|
||||
cuda::std::array<int, 3> dst_strides{1025 * 1024, 1024, 1};
|
||||
bench_copy(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_3D_odd_edges).set_name("transpose_3D_odd_edges (3D, int, 4GB)");
|
||||
|
||||
void transpose_src_small_15(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 3> shape{15, 2236962, 32};
|
||||
cuda::std::array<int, 3> src_strides{1, 15 * 32, 15};
|
||||
cuda::std::array<int, 3> dst_strides{2236962 * 32, 32, 1};
|
||||
bench_copy(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_src_small_15).set_name("transpose_src_small_15 (3D, int, 4GB)");
|
||||
|
||||
void transpose_src_small_16(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 3> shape{16, 2097152, 32};
|
||||
cuda::std::array<int, 3> src_strides{1, 16 * 32, 16};
|
||||
cuda::std::array<int, 3> dst_strides{2097152 * 32, 32, 1};
|
||||
bench_copy(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_src_small_16).set_name("transpose_src_small_16 (3D, int, 4GB)");
|
||||
|
||||
void transpose_src_small_17(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 3> shape{17, 1973790, 32};
|
||||
cuda::std::array<int, 3> src_strides{1, 17 * 32, 17};
|
||||
cuda::std::array<int, 3> dst_strides{1973790 * 32, 32, 1};
|
||||
bench_copy(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_src_small_17).set_name("transpose_src_small_17 (3D, int, 4GB)");
|
||||
|
||||
void transpose_dst_small_8_padded(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int64_t, 3> shape{32, 4194304, 8};
|
||||
cuda::std::array<int64_t, 3> src_strides{1, 32 * 8, 32};
|
||||
cuda::std::array<int64_t, 3> dst_strides{4194304 * 16, 16, 1};
|
||||
bench_copy<int, int64_t>(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_dst_small_8_padded).set_name("transpose_dst_small_8_padded (3D, int, 4GB)");
|
||||
|
||||
void transpose_dst_small_16_padded(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int64_t, 3> shape{32, 2097152, 16};
|
||||
cuda::std::array<int64_t, 3> src_strides{1, 32 * 16, 32};
|
||||
cuda::std::array<int64_t, 3> dst_strides{2097152 * 32, 32, 1};
|
||||
bench_copy<int, int64_t>(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_dst_small_16_padded).set_name("transpose_dst_small_16_padded (3D, int, 4GB)");
|
||||
|
||||
void transpose_src_small_16_4D(nvbench::state& state)
|
||||
{
|
||||
cuda::std::array<int, 4> shape{16, 1024, 2048, 32};
|
||||
cuda::std::array<int, 4> src_strides{1, 16 * 32, 16 * 32 * 1024, 16};
|
||||
cuda::std::array<int, 4> dst_strides{1024 * 2048 * 32, 32, 1024 * 32, 1};
|
||||
bench_copy(state, 0, shape, src_strides, 0, dst_strides);
|
||||
}
|
||||
NVBENCH_BENCH(transpose_src_small_16_4D).set_name("transpose_src_small_16_4D (4D, int, 4GB)");
|
||||
@@ -1,44 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include <nvbench/nvbench.cuh>
|
||||
#include <nvbench/range.cuh>
|
||||
|
||||
namespace cuda::experimental::cuco::benchmark::defaults
|
||||
{
|
||||
//! Key types covered by the default CUCO benchmark type axes.
|
||||
using key_type_range = ::nvbench::type_list<::nvbench::int32_t, ::nvbench::int64_t>;
|
||||
//! Value types covered by the default CUCO benchmark type axes.
|
||||
using value_type_range = ::nvbench::type_list<::nvbench::int32_t, ::nvbench::int64_t>;
|
||||
|
||||
//! Default number of inputs used when sweeping another benchmark axis.
|
||||
inline constexpr auto n = ::nvbench::int64_t{100'000'000};
|
||||
//! Default fixed-capacity map target occupancy.
|
||||
inline constexpr auto occupancy = 0.5;
|
||||
//! Default lookup matching rate for contains-style benchmarks.
|
||||
inline constexpr auto matching_rate = 1.0;
|
||||
//! Default deterministic seed used by benchmark data generators.
|
||||
inline constexpr auto seed = ::cuda::std::uint32_t{42};
|
||||
|
||||
//! Input-size sweep that remains cacheable for direct comparisons with CUCO benchmarks.
|
||||
inline const auto n_range_cache = ::std::vector<::nvbench::int64_t>{8'000, 80'000, 800'000, 8'000'000, 80'000'000};
|
||||
//! Occupancy sweep used by fixed-capacity container benchmarks.
|
||||
inline const auto occupancy_range = ::nvbench::range(0.1, 0.9, 0.1);
|
||||
//! Average multiplicity sweep for duplicate-key distributions.
|
||||
inline const auto multiplicity_range = ::std::vector<double>{1.0, 2.0, 4.0, 8.0, 16.0};
|
||||
//! Matching-rate sweep used by contains-style benchmarks.
|
||||
inline const auto matching_rate_range = ::nvbench::range(0.1, 1.0, 0.1);
|
||||
} // namespace cuda::experimental::cuco::benchmark::defaults
|
||||
@@ -1,285 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <thrust/execution_policy.h>
|
||||
#include <thrust/random.h>
|
||||
#include <thrust/sequence.h>
|
||||
#include <thrust/shuffle.h>
|
||||
#include <thrust/transform.h>
|
||||
|
||||
#include <cuda/iterator>
|
||||
#include <cuda/std/cmath>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/cstdint>
|
||||
#include <cuda/std/iterator>
|
||||
#include <cuda/std/limits>
|
||||
#include <cuda/std/type_traits>
|
||||
|
||||
#include <stdexcept>
|
||||
|
||||
#include "defaults.cuh"
|
||||
#include <nvbench/nvbench.cuh>
|
||||
|
||||
namespace cuda::experimental::cuco::benchmark
|
||||
{
|
||||
namespace distribution
|
||||
{
|
||||
//! Distribution tag for unique keys generated by shuffling the sequence `[0, N)`.
|
||||
struct unique
|
||||
{};
|
||||
|
||||
//! Distribution tag for uniformly sampled keys with controlled average multiplicity.
|
||||
struct uniform
|
||||
{
|
||||
//! Constructs a uniform distribution tag with the requested average key multiplicity.
|
||||
//!
|
||||
//! @param multiplicity Average number of generated keys that map to each unique key value.
|
||||
//! @throws std::invalid_argument if `multiplicity` is not finite or is less than 1.0.
|
||||
explicit uniform(double multiplicity)
|
||||
: multiplicity{multiplicity}
|
||||
{
|
||||
if (!cuda::std::isfinite(multiplicity) || multiplicity < 1.0)
|
||||
{
|
||||
throw ::std::invalid_argument{"Multiplicity must be finite and at least 1"};
|
||||
}
|
||||
}
|
||||
|
||||
//! Average number of generated keys that map to each unique key value.
|
||||
double multiplicity;
|
||||
};
|
||||
} // namespace distribution
|
||||
|
||||
namespace detail
|
||||
{
|
||||
template <typename Key, typename Distribution, typename Rng>
|
||||
struct generate_uniform_fn
|
||||
{
|
||||
__host__ __device__ constexpr generate_uniform_fn(cuda::std::size_t num, Distribution dist, cuda::std::size_t seed)
|
||||
: num{num}
|
||||
, dist{dist}
|
||||
, seed{seed}
|
||||
{}
|
||||
|
||||
__host__ __device__ constexpr Key operator()(cuda::std::size_t idx) const noexcept
|
||||
{
|
||||
Rng rng;
|
||||
rng.seed(seed + idx * 1664525ull + 1013904223ull);
|
||||
|
||||
const auto num_unique_keys_unclamped =
|
||||
static_cast<cuda::std::size_t>(cuda::std::ceil(static_cast<double>(num) / dist.multiplicity));
|
||||
const auto num_unique_keys =
|
||||
num_unique_keys_unclamped < cuda::std::size_t{1} ? cuda::std::size_t{1} : num_unique_keys_unclamped;
|
||||
thrust::uniform_int_distribution<Key> key_dist{Key{0}, static_cast<Key>(num_unique_keys - 1)};
|
||||
return key_dist(rng);
|
||||
}
|
||||
|
||||
cuda::std::size_t num;
|
||||
Distribution dist;
|
||||
cuda::std::size_t seed;
|
||||
};
|
||||
|
||||
template <typename Key, typename Rng>
|
||||
struct dropout_fn
|
||||
{
|
||||
__host__ __device__ constexpr explicit dropout_fn(cuda::std::size_t num)
|
||||
: num{num}
|
||||
{}
|
||||
|
||||
__host__ __device__ Key operator()(cuda::std::size_t seed) const noexcept
|
||||
{
|
||||
Rng rng;
|
||||
thrust::uniform_int_distribution<Key> dist{static_cast<Key>(num), cuda::std::numeric_limits<Key>::max()};
|
||||
rng.seed(seed);
|
||||
return dist(rng);
|
||||
}
|
||||
|
||||
cuda::std::size_t num;
|
||||
};
|
||||
|
||||
template <typename Rng>
|
||||
struct dropout_pred
|
||||
{
|
||||
__host__ __device__ constexpr explicit dropout_pred(double keep_prob)
|
||||
: keep_prob{keep_prob}
|
||||
{}
|
||||
|
||||
__host__ __device__ bool operator()(cuda::std::size_t seed) const noexcept
|
||||
{
|
||||
Rng rng;
|
||||
thrust::uniform_real_distribution<double> dist{0.0, 1.0};
|
||||
rng.seed(seed);
|
||||
return dist(rng) > keep_prob;
|
||||
}
|
||||
|
||||
double keep_prob;
|
||||
};
|
||||
} // namespace detail
|
||||
|
||||
//! Random key generator used by CUCO benchmarks.
|
||||
//!
|
||||
//! The generator defaults to `defaults::seed` to keep benchmark data reproducible across runs.
|
||||
//!
|
||||
//! @tparam Rng Pseudo-random number generator type compatible with Thrust random distributions.
|
||||
template <typename Rng = thrust::default_random_engine>
|
||||
class key_generator
|
||||
{
|
||||
public:
|
||||
//! Constructs a key generator with the given seed.
|
||||
//!
|
||||
//! @param seed Seed used to initialize the generator state.
|
||||
explicit key_generator(cuda::std::uint32_t seed = defaults::seed)
|
||||
: rng{seed}
|
||||
{}
|
||||
|
||||
//! Generates keys according to the given distribution using the default device execution policy.
|
||||
//!
|
||||
//! @tparam Distribution Distribution tag type.
|
||||
//! @tparam OutputIt Output iterator type whose value type is the generated key type.
|
||||
//! @param dist Distribution tag controlling how keys are generated.
|
||||
//! @param out_begin Beginning of the output key range.
|
||||
//! @param out_end End of the output key range.
|
||||
//! @throws std::invalid_argument if `Distribution` is not a supported distribution tag.
|
||||
template <typename Distribution, typename OutputIt>
|
||||
void generate(Distribution dist, OutputIt out_begin, OutputIt out_end)
|
||||
{
|
||||
generate(dist, out_begin, out_end, thrust::device);
|
||||
}
|
||||
|
||||
//! Generates keys according to the given distribution using the provided execution policy.
|
||||
//!
|
||||
//! @tparam Distribution Distribution tag type.
|
||||
//! @tparam OutputIt Output iterator type whose value type is the generated key type.
|
||||
//! @tparam ExecPolicy Thrust execution policy type.
|
||||
//! @param dist Distribution tag controlling how keys are generated.
|
||||
//! @param out_begin Beginning of the output key range.
|
||||
//! @param out_end End of the output key range.
|
||||
//! @param exec_policy Execution policy used for the underlying Thrust algorithms.
|
||||
//! @throws std::invalid_argument if `Distribution` is not a supported distribution tag.
|
||||
template <typename Distribution, typename OutputIt, typename ExecPolicy>
|
||||
void generate(Distribution dist, OutputIt out_begin, OutputIt out_end, ExecPolicy exec_policy)
|
||||
{
|
||||
using value_type = typename cuda::std::iterator_traits<OutputIt>::value_type;
|
||||
|
||||
if constexpr (cuda::std::is_same_v<Distribution, distribution::unique>)
|
||||
{
|
||||
thrust::sequence(exec_policy, out_begin, out_end, value_type{0});
|
||||
thrust::shuffle(exec_policy, out_begin, out_end, rng);
|
||||
}
|
||||
else if constexpr (cuda::std::is_same_v<Distribution, distribution::uniform>)
|
||||
{
|
||||
const auto num_keys = static_cast<cuda::std::size_t>(cuda::std::distance(out_begin, out_end));
|
||||
const auto seed = static_cast<cuda::std::size_t>(rng());
|
||||
|
||||
thrust::transform(
|
||||
exec_policy,
|
||||
cuda::counting_iterator<cuda::std::size_t>{0},
|
||||
cuda::counting_iterator<cuda::std::size_t>{num_keys},
|
||||
out_begin,
|
||||
detail::generate_uniform_fn<value_type, Distribution, Rng>{num_keys, dist, seed});
|
||||
}
|
||||
else
|
||||
{
|
||||
throw ::std::invalid_argument{"Unexpected distribution type"};
|
||||
}
|
||||
}
|
||||
|
||||
//! Drops keys with probability `1 - keep_prob` using the default device execution policy.
|
||||
//!
|
||||
//! Replaced keys are sampled from `[N, max_key]`, where `N` is the number of keys in the range.
|
||||
//!
|
||||
//! The full range is shuffled afterward, even when all keys are kept.
|
||||
//!
|
||||
//! @tparam InOutIt Mutable iterator type whose value type is the key type.
|
||||
//! @param begin Beginning of the key range to update in place.
|
||||
//! @param end End of the key range to update in place.
|
||||
//! @param keep_prob Probability of keeping each original key. Must be in `[0, 1]`.
|
||||
//! @throws std::invalid_argument if `keep_prob` is outside `[0, 1]`.
|
||||
template <typename InOutIt>
|
||||
void dropout(InOutIt begin, InOutIt end, double keep_prob)
|
||||
{
|
||||
dropout(begin, end, keep_prob, thrust::device);
|
||||
}
|
||||
|
||||
//! Drops keys with probability `1 - keep_prob` using the provided execution policy.
|
||||
//!
|
||||
//! Replaced keys are sampled from `[N, max_key]`, where `N` is the number of keys in the range.
|
||||
//!
|
||||
//! The full range is shuffled afterward, even when all keys are kept.
|
||||
//!
|
||||
//! @tparam InOutIt Mutable iterator type whose value type is the key type.
|
||||
//! @tparam ExecPolicy Thrust execution policy type.
|
||||
//! @param begin Beginning of the key range to update in place.
|
||||
//! @param end End of the key range to update in place.
|
||||
//! @param keep_prob Probability of keeping each original key. Must be in `[0, 1]`.
|
||||
//! @param exec_policy Execution policy used for the underlying Thrust algorithms.
|
||||
//! @throws std::invalid_argument if `keep_prob` is outside `[0, 1]`.
|
||||
template <typename InOutIt, typename ExecPolicy>
|
||||
void dropout(InOutIt begin, InOutIt end, double keep_prob, ExecPolicy exec_policy)
|
||||
{
|
||||
using value_type = typename cuda::std::iterator_traits<InOutIt>::value_type;
|
||||
|
||||
if (keep_prob < 0.0 || keep_prob > 1.0)
|
||||
{
|
||||
throw ::std::invalid_argument{"Probability needs to be between 0 and 1"};
|
||||
}
|
||||
|
||||
if (keep_prob < 1.0)
|
||||
{
|
||||
const auto num_keys = static_cast<cuda::std::size_t>(cuda::std::distance(begin, end));
|
||||
cuda::counting_iterator<cuda::std::size_t> seeds{static_cast<cuda::std::size_t>(rng())};
|
||||
|
||||
thrust::transform_if(
|
||||
exec_policy,
|
||||
seeds,
|
||||
seeds + num_keys,
|
||||
begin,
|
||||
detail::dropout_fn<value_type, Rng>{num_keys},
|
||||
detail::dropout_pred<Rng>{keep_prob});
|
||||
}
|
||||
|
||||
thrust::shuffle(exec_policy, begin, end, rng);
|
||||
}
|
||||
|
||||
private:
|
||||
Rng rng;
|
||||
};
|
||||
|
||||
//! Constructs the requested distribution tag from NVBench axis values.
|
||||
//!
|
||||
//! `distribution::uniform` reads the `Multiplicity` axis from `state`.
|
||||
//!
|
||||
//! @tparam Distribution Distribution tag type to construct.
|
||||
//! @param state NVBench state containing distribution-specific axis values.
|
||||
//! @return Distribution tag initialized from the benchmark state.
|
||||
//! @throws std::invalid_argument if `Distribution` is not a supported distribution tag.
|
||||
template <typename Distribution>
|
||||
Distribution dist_from_state(nvbench::state const& state)
|
||||
{
|
||||
if constexpr (cuda::std::is_same_v<Distribution, distribution::unique>)
|
||||
{
|
||||
return Distribution{};
|
||||
}
|
||||
else if constexpr (cuda::std::is_same_v<Distribution, distribution::uniform>)
|
||||
{
|
||||
return Distribution{state.get_float64("Multiplicity")};
|
||||
}
|
||||
else
|
||||
{
|
||||
throw ::std::invalid_argument{"Unexpected distribution type"};
|
||||
}
|
||||
}
|
||||
} // namespace cuda::experimental::cuco::benchmark
|
||||
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(
|
||||
cuda::experimental::cuco::benchmark::distribution::unique, "UNIQUE", "distribution::unique");
|
||||
NVBENCH_DECLARE_TYPE_STRINGS(
|
||||
cuda::experimental::cuco::benchmark::distribution::uniform, "UNIFORM", "distribution::uniform");
|
||||
@@ -1,110 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <thrust/execution_policy.h>
|
||||
#include <thrust/transform.h>
|
||||
|
||||
#include <cuda/buffer>
|
||||
#include <cuda/memory_resource>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/functional>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/__cuco/fixed_capacity_map.cuh>
|
||||
#include <cuda/experimental/__cuco/types.cuh>
|
||||
|
||||
#include "../common/defaults.cuh"
|
||||
#include "../common/key_generator.cuh"
|
||||
#include <nvbench/nvbench.cuh>
|
||||
|
||||
namespace cudax = cuda::experimental;
|
||||
namespace bench = cudax::cuco::benchmark;
|
||||
|
||||
/**
|
||||
* @brief A benchmark evaluating `cudax::cuco::fixed_capacity_map::contains_async` performance.
|
||||
*/
|
||||
template <typename Key, typename Value, typename Dist>
|
||||
void fixed_capacity_map_contains(nvbench::state& state, nvbench::type_list<Key, Value, Dist>)
|
||||
{
|
||||
if constexpr (sizeof(Key) != sizeof(Value))
|
||||
{
|
||||
state.skip("Key and Value must have the same size.");
|
||||
}
|
||||
else
|
||||
{
|
||||
using pair_type = cuda::std::pair<Key, Value>;
|
||||
using map_type = cudax::cuco::fixed_capacity_map<Key, Value>;
|
||||
|
||||
const auto num_keys = state.get_int64("NumInputs");
|
||||
const auto occupancy = state.get_float64("Occupancy");
|
||||
const auto matching_rate = state.get_float64("MatchingRate");
|
||||
|
||||
const auto size = static_cast<cuda::std::size_t>(static_cast<double>(num_keys) / occupancy);
|
||||
|
||||
const auto device = cuda::device_ref{0};
|
||||
cuda::stream stream{device};
|
||||
const cuda::device_memory_pool_ref mr = cuda::device_default_memory_pool(device);
|
||||
const auto exec_policy = thrust::cuda::par_nosync.on(stream.get());
|
||||
|
||||
auto keys = cuda::make_device_buffer<Key>(stream, device, num_keys, cuda::no_init);
|
||||
|
||||
bench::key_generator gen{};
|
||||
gen.generate(bench::dist_from_state<Dist>(state), keys.begin(), keys.end(), exec_policy);
|
||||
|
||||
auto pairs = cuda::make_device_buffer<pair_type>(stream, device, num_keys, cuda::no_init);
|
||||
thrust::transform(exec_policy, keys.begin(), keys.end(), pairs.begin(), [] __device__(Key const& key) {
|
||||
return pair_type{key, Value{}};
|
||||
});
|
||||
|
||||
map_type map{stream, mr, size, cudax::cuco::empty_key(Key{-1}), cudax::cuco::empty_value(Value{-1})};
|
||||
map.insert(stream, pairs.begin(), pairs.end());
|
||||
|
||||
gen.dropout(keys.begin(), keys.end(), matching_rate, exec_policy);
|
||||
|
||||
auto result = cuda::make_device_buffer<bool>(stream, device, num_keys, cuda::no_init);
|
||||
stream.sync();
|
||||
|
||||
state.add_element_count(num_keys);
|
||||
state.exec([&](nvbench::launch& launch) {
|
||||
map.contains_async({launch.get_stream()}, keys.begin(), keys.end(), result.begin());
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
NVBENCH_BENCH_TYPES(fixed_capacity_map_contains,
|
||||
NVBENCH_TYPE_AXES(bench::defaults::key_type_range,
|
||||
bench::defaults::value_type_range,
|
||||
nvbench::type_list<bench::distribution::unique>))
|
||||
.set_name("fixed_capacity_map_contains_unique_capacity")
|
||||
.set_type_axes_names({"Key", "Value", "Distribution"})
|
||||
.add_int64_axis("NumInputs", bench::defaults::n_range_cache)
|
||||
.add_float64_axis("Occupancy", {bench::defaults::occupancy})
|
||||
.add_float64_axis("MatchingRate", {bench::defaults::matching_rate});
|
||||
|
||||
NVBENCH_BENCH_TYPES(fixed_capacity_map_contains,
|
||||
NVBENCH_TYPE_AXES(bench::defaults::key_type_range,
|
||||
bench::defaults::value_type_range,
|
||||
nvbench::type_list<bench::distribution::unique>))
|
||||
.set_name("fixed_capacity_map_contains_unique_occupancy")
|
||||
.set_type_axes_names({"Key", "Value", "Distribution"})
|
||||
.add_int64_axis("NumInputs", {bench::defaults::n})
|
||||
.add_float64_axis("Occupancy", bench::defaults::occupancy_range)
|
||||
.add_float64_axis("MatchingRate", {bench::defaults::matching_rate});
|
||||
|
||||
NVBENCH_BENCH_TYPES(fixed_capacity_map_contains,
|
||||
NVBENCH_TYPE_AXES(bench::defaults::key_type_range,
|
||||
bench::defaults::value_type_range,
|
||||
nvbench::type_list<bench::distribution::unique>))
|
||||
.set_name("fixed_capacity_map_contains_unique_matching_rate")
|
||||
.set_type_axes_names({"Key", "Value", "Distribution"})
|
||||
.add_int64_axis("NumInputs", {bench::defaults::n})
|
||||
.add_float64_axis("Occupancy", {bench::defaults::occupancy})
|
||||
.add_float64_axis("MatchingRate", bench::defaults::matching_rate_range);
|
||||
@@ -1,108 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <thrust/execution_policy.h>
|
||||
#include <thrust/transform.h>
|
||||
|
||||
#include <cuda/buffer>
|
||||
#include <cuda/memory_resource>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/functional>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/__cuco/fixed_capacity_map.cuh>
|
||||
#include <cuda/experimental/__cuco/types.cuh>
|
||||
|
||||
#include "../common/defaults.cuh"
|
||||
#include "../common/key_generator.cuh"
|
||||
#include <nvbench/nvbench.cuh>
|
||||
|
||||
namespace cudax = cuda::experimental;
|
||||
namespace bench = cudax::cuco::benchmark;
|
||||
|
||||
/**
|
||||
* @brief A benchmark evaluating `cudax::cuco::fixed_capacity_map::find_async` performance.
|
||||
*/
|
||||
template <typename Key, typename Value, typename Dist>
|
||||
void fixed_capacity_map_find(nvbench::state& state, nvbench::type_list<Key, Value, Dist>)
|
||||
{
|
||||
if constexpr (sizeof(Key) != sizeof(Value))
|
||||
{
|
||||
state.skip("Key and Value must have the same size.");
|
||||
}
|
||||
else
|
||||
{
|
||||
using pair_type = cuda::std::pair<Key, Value>;
|
||||
using map_type = cudax::cuco::fixed_capacity_map<Key, Value>;
|
||||
|
||||
const auto num_keys = static_cast<::cuda::std::size_t>(state.get_int64("NumInputs"));
|
||||
const auto occupancy = state.get_float64("Occupancy");
|
||||
const auto matching_rate = state.get_float64("MatchingRate");
|
||||
|
||||
const auto device = cuda::device_ref{0};
|
||||
cuda::stream stream{device};
|
||||
const cuda::device_memory_pool_ref mr = cuda::device_default_memory_pool(device);
|
||||
const auto exec_policy = thrust::cuda::par_nosync.on(stream.get());
|
||||
|
||||
auto keys = cuda::make_device_buffer<Key>(stream, device, num_keys, cuda::no_init);
|
||||
|
||||
bench::key_generator gen{};
|
||||
gen.generate(bench::dist_from_state<Dist>(state), keys.begin(), keys.end(), exec_policy);
|
||||
|
||||
auto pairs = cuda::make_device_buffer<pair_type>(stream, device, num_keys, cuda::no_init);
|
||||
thrust::transform(exec_policy, keys.begin(), keys.end(), pairs.begin(), [] __device__(Key const& key) {
|
||||
return pair_type{key, Value{}};
|
||||
});
|
||||
|
||||
map_type map{stream, mr, num_keys, occupancy, cudax::cuco::empty_key(Key{-1}), cudax::cuco::empty_value(Value{-1})};
|
||||
map.insert(stream, pairs.begin(), pairs.end());
|
||||
|
||||
gen.dropout(keys.begin(), keys.end(), matching_rate, exec_policy);
|
||||
|
||||
auto result = cuda::make_device_buffer<Value>(stream, device, num_keys, cuda::no_init);
|
||||
stream.sync();
|
||||
|
||||
state.add_element_count(num_keys);
|
||||
state.exec([&](nvbench::launch& launch) {
|
||||
map.find_async({launch.get_stream()}, keys.begin(), keys.end(), result.begin());
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
NVBENCH_BENCH_TYPES(fixed_capacity_map_find,
|
||||
NVBENCH_TYPE_AXES(bench::defaults::key_type_range,
|
||||
bench::defaults::value_type_range,
|
||||
nvbench::type_list<bench::distribution::unique>))
|
||||
.set_name("fixed_capacity_map_find_unique_capacity")
|
||||
.set_type_axes_names({"Key", "Value", "Distribution"})
|
||||
.add_int64_axis("NumInputs", bench::defaults::n_range_cache)
|
||||
.add_float64_axis("Occupancy", {bench::defaults::occupancy})
|
||||
.add_float64_axis("MatchingRate", {bench::defaults::matching_rate});
|
||||
|
||||
NVBENCH_BENCH_TYPES(fixed_capacity_map_find,
|
||||
NVBENCH_TYPE_AXES(bench::defaults::key_type_range,
|
||||
bench::defaults::value_type_range,
|
||||
nvbench::type_list<bench::distribution::unique>))
|
||||
.set_name("fixed_capacity_map_find_unique_occupancy")
|
||||
.set_type_axes_names({"Key", "Value", "Distribution"})
|
||||
.add_int64_axis("NumInputs", {bench::defaults::n})
|
||||
.add_float64_axis("Occupancy", bench::defaults::occupancy_range)
|
||||
.add_float64_axis("MatchingRate", {bench::defaults::matching_rate});
|
||||
|
||||
NVBENCH_BENCH_TYPES(fixed_capacity_map_find,
|
||||
NVBENCH_TYPE_AXES(bench::defaults::key_type_range,
|
||||
bench::defaults::value_type_range,
|
||||
nvbench::type_list<bench::distribution::unique>))
|
||||
.set_name("fixed_capacity_map_find_unique_matching_rate")
|
||||
.set_type_axes_names({"Key", "Value", "Distribution"})
|
||||
.add_int64_axis("NumInputs", {bench::defaults::n})
|
||||
.add_float64_axis("Occupancy", {bench::defaults::occupancy})
|
||||
.add_float64_axis("MatchingRate", bench::defaults::matching_rate_range);
|
||||
@@ -1,105 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <thrust/execution_policy.h>
|
||||
#include <thrust/transform.h>
|
||||
|
||||
#include <cuda/buffer>
|
||||
#include <cuda/memory_resource>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/functional>
|
||||
#include <cuda/std/utility>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/__cuco/fixed_capacity_map.cuh>
|
||||
#include <cuda/experimental/__cuco/types.cuh>
|
||||
|
||||
#include "../common/defaults.cuh"
|
||||
#include "../common/key_generator.cuh"
|
||||
#include <nvbench/nvbench.cuh>
|
||||
|
||||
namespace cudax = cuda::experimental;
|
||||
namespace bench = cudax::cuco::benchmark;
|
||||
|
||||
/**
|
||||
* @brief A benchmark evaluating `cudax::cuco::fixed_capacity_map::insert_async` performance.
|
||||
*/
|
||||
template <typename Key, typename Value, typename Dist>
|
||||
void fixed_capacity_map_insert(nvbench::state& state, nvbench::type_list<Key, Value, Dist>)
|
||||
{
|
||||
if constexpr (sizeof(Key) != sizeof(Value))
|
||||
{
|
||||
state.skip("Key and Value must have the same size.");
|
||||
}
|
||||
else
|
||||
{
|
||||
using pair_type = cuda::std::pair<Key, Value>;
|
||||
using map_type = cudax::cuco::fixed_capacity_map<Key, Value>;
|
||||
|
||||
const auto num_keys = state.get_int64("NumInputs");
|
||||
const auto occupancy = state.get_float64("Occupancy");
|
||||
|
||||
const auto size = static_cast<cuda::std::size_t>(static_cast<double>(num_keys) / occupancy);
|
||||
|
||||
const auto device = cuda::device_ref{0};
|
||||
cuda::stream stream{device};
|
||||
const cuda::device_memory_pool_ref mr = cuda::device_default_memory_pool(device);
|
||||
const auto exec_policy = thrust::cuda::par_nosync.on(stream.get());
|
||||
|
||||
auto keys = cuda::make_device_buffer<Key>(stream, device, num_keys, cuda::no_init);
|
||||
|
||||
bench::key_generator gen{};
|
||||
gen.generate(bench::dist_from_state<Dist>(state), keys.begin(), keys.end(), exec_policy);
|
||||
|
||||
auto pairs = cuda::make_device_buffer<pair_type>(stream, device, num_keys, cuda::no_init);
|
||||
thrust::transform(exec_policy, keys.begin(), keys.end(), pairs.begin(), [] __device__(Key const& key) {
|
||||
return pair_type{key, Value{}};
|
||||
});
|
||||
|
||||
map_type map{stream, mr, size, cudax::cuco::empty_key(Key{-1}), cudax::cuco::empty_value(Value{-1})};
|
||||
stream.sync();
|
||||
|
||||
state.add_element_count(num_keys);
|
||||
state.exec(nvbench::exec_tag::timer, [&](nvbench::launch& launch, auto& timer) {
|
||||
timer.start();
|
||||
map.insert_async({launch.get_stream()}, pairs.begin(), pairs.end());
|
||||
timer.stop();
|
||||
map.clear_async({launch.get_stream()});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
NVBENCH_BENCH_TYPES(fixed_capacity_map_insert,
|
||||
NVBENCH_TYPE_AXES(bench::defaults::key_type_range,
|
||||
bench::defaults::value_type_range,
|
||||
nvbench::type_list<bench::distribution::unique>))
|
||||
.set_name("fixed_capacity_map_insert_unique_capacity")
|
||||
.set_type_axes_names({"Key", "Value", "Distribution"})
|
||||
.add_int64_axis("NumInputs", bench::defaults::n_range_cache)
|
||||
.add_float64_axis("Occupancy", {bench::defaults::occupancy});
|
||||
|
||||
NVBENCH_BENCH_TYPES(fixed_capacity_map_insert,
|
||||
NVBENCH_TYPE_AXES(bench::defaults::key_type_range,
|
||||
bench::defaults::value_type_range,
|
||||
nvbench::type_list<bench::distribution::unique>))
|
||||
.set_name("fixed_capacity_map_insert_unique_occupancy")
|
||||
.set_type_axes_names({"Key", "Value", "Distribution"})
|
||||
.add_int64_axis("NumInputs", {bench::defaults::n})
|
||||
.add_float64_axis("Occupancy", bench::defaults::occupancy_range);
|
||||
|
||||
NVBENCH_BENCH_TYPES(fixed_capacity_map_insert,
|
||||
NVBENCH_TYPE_AXES(bench::defaults::key_type_range,
|
||||
bench::defaults::value_type_range,
|
||||
nvbench::type_list<bench::distribution::uniform>))
|
||||
.set_name("fixed_capacity_map_insert_uniform_multiplicity")
|
||||
.set_type_axes_names({"Key", "Value", "Distribution"})
|
||||
.add_int64_axis("NumInputs", {bench::defaults::n})
|
||||
.add_float64_axis("Occupancy", {bench::defaults::occupancy})
|
||||
.add_float64_axis("Multiplicity", bench::defaults::multiplicity_range);
|
||||
@@ -1,131 +0,0 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved.
|
||||
// SPDX-License-Identifier: SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
|
||||
#include <thrust/device_vector.h>
|
||||
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/std/cstdint>
|
||||
|
||||
#include <cuda/experimental/__cuco/hash_functions.cuh>
|
||||
|
||||
#include <nvbench/nvbench.cuh>
|
||||
#include <nvbench/range.cuh>
|
||||
|
||||
namespace cudax = cuda::experimental;
|
||||
|
||||
// repeat hash computation n times
|
||||
static constexpr auto n_repeats = 100;
|
||||
|
||||
template <cuda::std::int32_t Words>
|
||||
struct large_key
|
||||
{
|
||||
constexpr __host__ __device__ large_key(cuda::std::int32_t seed) noexcept
|
||||
{
|
||||
for (cuda::std::int32_t i = 0; i < Words; ++i)
|
||||
{
|
||||
data_[i] = seed;
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
cuda::std::int32_t data_[Words];
|
||||
};
|
||||
|
||||
template <cuda::std::int32_t BlockSize, typename Key, typename Hasher, typename OutputIt>
|
||||
__global__ void hash_bench_kernel(Hasher hash, size_t n, OutputIt out, bool materialize_result)
|
||||
{
|
||||
size_t const gid = static_cast<size_t>(BlockSize) * blockIdx.x + threadIdx.x;
|
||||
size_t const loop_stride = static_cast<size_t>(gridDim.x) * BlockSize;
|
||||
size_t idx = gid;
|
||||
using result_t = decltype(hash(0));
|
||||
|
||||
result_t agg{};
|
||||
|
||||
while (idx < n)
|
||||
{
|
||||
Key key(idx);
|
||||
for (cuda::std::int32_t i = 0; i < n_repeats; ++i)
|
||||
{ // execute hash func n times
|
||||
agg += hash(key);
|
||||
}
|
||||
idx += loop_stride;
|
||||
}
|
||||
|
||||
if (materialize_result)
|
||||
{
|
||||
out[gid] = agg;
|
||||
}
|
||||
}
|
||||
|
||||
// benchmark evaluating performance of various hash functions
|
||||
template <typename HasherTag, typename Key>
|
||||
void hash_eval(nvbench::state& state, nvbench::type_list<HasherTag, Key>)
|
||||
{
|
||||
using Hash = typename HasherTag::template fn<Key>;
|
||||
|
||||
bool const materialize_result = false;
|
||||
constexpr auto block_size = 128;
|
||||
auto const num_keys = state.get_int64("NumInputs");
|
||||
auto const grid_size = (num_keys + block_size * 16 - 1) / block_size * 16;
|
||||
using result_t = decltype(std::declval<Hash>()(std::declval<cuda::std::int32_t>()));
|
||||
|
||||
thrust::device_vector<result_t> hash_values((materialize_result) ? num_keys : 1);
|
||||
|
||||
state.add_element_count(num_keys);
|
||||
|
||||
state.exec([&](nvbench::launch& launch) {
|
||||
hash_bench_kernel<block_size, Key>
|
||||
<<<grid_size, block_size, 0, launch.get_stream()>>>(Hash{}, num_keys, hash_values.begin(), materialize_result);
|
||||
});
|
||||
}
|
||||
|
||||
struct xxhash_32_tag
|
||||
{
|
||||
template <typename Key>
|
||||
using fn = cudax::cuco::hash<Key, cudax::cuco::hash_algorithm::xxhash_32>;
|
||||
};
|
||||
|
||||
struct xxhash_64_tag
|
||||
{
|
||||
template <typename Key>
|
||||
using fn = cudax::cuco::hash<Key, cudax::cuco::hash_algorithm::xxhash_64>;
|
||||
};
|
||||
|
||||
struct murmurhash3_32_tag
|
||||
{
|
||||
template <typename Key>
|
||||
using fn = cudax::cuco::hash<Key, cudax::cuco::hash_algorithm::murmurhash3_32>;
|
||||
};
|
||||
|
||||
#if _CCCL_HAS_INT128()
|
||||
|
||||
struct murmurhash3_x86_128_tag
|
||||
{
|
||||
template <typename Key>
|
||||
using fn = cudax::cuco::hash<Key, cudax::cuco::hash_algorithm::murmurhash3_x86_128>;
|
||||
};
|
||||
|
||||
struct murmurhash3_x64_128_tag
|
||||
{
|
||||
template <typename Key>
|
||||
using fn = cudax::cuco::hash<Key, cudax::cuco::hash_algorithm::murmurhash3_x64_128>;
|
||||
};
|
||||
|
||||
#endif // _CCCL_HAS_INT128()
|
||||
|
||||
NVBENCH_BENCH_TYPES(
|
||||
hash_eval,
|
||||
NVBENCH_TYPE_AXES(
|
||||
nvbench::type_list<xxhash_32_tag,
|
||||
xxhash_64_tag,
|
||||
murmurhash3_32_tag
|
||||
#if _CCCL_HAS_INT128()
|
||||
,
|
||||
murmurhash3_x86_128_tag,
|
||||
murmurhash3_x64_128_tag
|
||||
#endif // _CCCL_HAS_INT128()
|
||||
>,
|
||||
nvbench::type_list<cuda::std::int32_t, large_key<4>, large_key<8>, large_key<16>, large_key<32>>))
|
||||
.set_name("hash_function_eval")
|
||||
.set_type_axes_names({"Hash", "Key"})
|
||||
.add_int64_power_of_two_axis("NumInputs", nvbench::range(18, 26, 4));
|
||||
@@ -1,133 +0,0 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of CUDA Experimental in CUDA C++ Core Libraries,
|
||||
// under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#include <thrust/execution_policy.h>
|
||||
#include <thrust/sequence.h>
|
||||
|
||||
#include <cuda/buffer>
|
||||
#include <cuda/memory_resource>
|
||||
#include <cuda/std/cmath>
|
||||
#include <cuda/std/cstddef>
|
||||
#include <cuda/stream>
|
||||
|
||||
#include <cuda/experimental/__cuco/hyperloglog.cuh>
|
||||
|
||||
#include "common/defaults.cuh"
|
||||
#include <nvbench/nvbench.cuh>
|
||||
|
||||
namespace cudax = cuda::experimental;
|
||||
namespace bench = cudax::cuco::benchmark;
|
||||
|
||||
namespace
|
||||
{
|
||||
template <typename Key>
|
||||
void add_relative_error_summary(
|
||||
nvbench::state& state,
|
||||
cudax::cuco::hyperloglog<Key>& estimator,
|
||||
cuda::stream_ref stream,
|
||||
Key* first,
|
||||
cuda::std::size_t num_items)
|
||||
{
|
||||
estimator.add(stream, first, first + num_items);
|
||||
const auto estimated_cardinality = estimator.estimate(stream);
|
||||
const auto relative_error =
|
||||
cuda::std::abs(static_cast<double>(estimated_cardinality) / static_cast<double>(num_items) - 1.0);
|
||||
estimator.clear(stream);
|
||||
|
||||
auto& summary = state.add_summary("RelativeError");
|
||||
summary.set_string("hint", "RelErr");
|
||||
summary.set_string("short_name", "RelativeError");
|
||||
summary.set_string("description", "Relative approximation error.");
|
||||
summary.set_float64("value", relative_error);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
/**
|
||||
* @brief A benchmark evaluating `cudax::cuco::hyperloglog` end-to-end performance.
|
||||
*/
|
||||
template <typename Key>
|
||||
void hyperloglog_e2e(nvbench::state& state, nvbench::type_list<Key>)
|
||||
{
|
||||
using estimator_type = cudax::cuco::hyperloglog<Key>;
|
||||
using sketch_size_kb_type = typename estimator_type::sketch_size_kb;
|
||||
|
||||
const auto num_items = static_cast<cuda::std::size_t>(state.get_int64("NumInputs"));
|
||||
const auto sketch_size_kb = sketch_size_kb_type{static_cast<double>(state.get_int64("SketchSizeKB"))};
|
||||
|
||||
const auto device = cuda::device_ref{0};
|
||||
cuda::stream stream{device};
|
||||
const cuda::device_memory_pool_ref mr = cuda::device_default_memory_pool(device);
|
||||
|
||||
auto items = cuda::make_device_buffer<Key>(stream, device, num_items, cuda::no_init);
|
||||
thrust::sequence(thrust::cuda::par_nosync.on(stream.get()), items.begin(), items.end(), Key{0});
|
||||
|
||||
estimator_type estimator{stream, mr, sketch_size_kb};
|
||||
stream.sync();
|
||||
|
||||
state.add_element_count(num_items);
|
||||
state.add_global_memory_reads<Key>(num_items, "InputSize");
|
||||
|
||||
add_relative_error_summary(state, estimator, stream, items.data(), num_items);
|
||||
|
||||
state.exec(nvbench::exec_tag::sync | nvbench::exec_tag::timer, [&](nvbench::launch& launch, auto& timer) {
|
||||
timer.start();
|
||||
estimator.add_async({launch.get_stream()}, items.begin(), items.end());
|
||||
[[maybe_unused]] const auto estimated_cardinality = estimator.estimate({launch.get_stream()});
|
||||
timer.stop();
|
||||
|
||||
estimator.clear_async({launch.get_stream()});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief A benchmark evaluating `cudax::cuco::hyperloglog::add_async` performance.
|
||||
*/
|
||||
template <typename Key>
|
||||
void hyperloglog_add(nvbench::state& state, nvbench::type_list<Key>)
|
||||
{
|
||||
using estimator_type = cudax::cuco::hyperloglog<Key>;
|
||||
using sketch_size_kb_type = typename estimator_type::sketch_size_kb;
|
||||
|
||||
const auto num_items = static_cast<cuda::std::size_t>(state.get_int64("NumInputs"));
|
||||
const auto sketch_size_kb = sketch_size_kb_type{static_cast<double>(state.get_int64("SketchSizeKB"))};
|
||||
|
||||
const auto device = cuda::device_ref{0};
|
||||
cuda::stream stream{device};
|
||||
const cuda::device_memory_pool_ref mr = cuda::device_default_memory_pool(device);
|
||||
|
||||
auto items = cuda::make_device_buffer<Key>(stream, device, num_items, cuda::no_init);
|
||||
thrust::sequence(thrust::cuda::par_nosync.on(stream.get()), items.begin(), items.end(), Key{0});
|
||||
|
||||
estimator_type estimator{stream, mr, sketch_size_kb};
|
||||
stream.sync();
|
||||
|
||||
state.add_element_count(num_items);
|
||||
state.add_global_memory_reads<Key>(num_items, "InputSize");
|
||||
|
||||
state.exec(nvbench::exec_tag::timer, [&](nvbench::launch& launch, auto& timer) {
|
||||
timer.start();
|
||||
estimator.add_async({launch.get_stream()}, items.begin(), items.end());
|
||||
timer.stop();
|
||||
|
||||
estimator.clear_async({launch.get_stream()});
|
||||
});
|
||||
}
|
||||
|
||||
NVBENCH_BENCH_TYPES(hyperloglog_e2e, NVBENCH_TYPE_AXES(bench::defaults::key_type_range))
|
||||
.set_name("hyperloglog_e2e")
|
||||
.set_type_axes_names({"Key"})
|
||||
.add_int64_power_of_two_axis("NumInputs", {30})
|
||||
.add_int64_axis("SketchSizeKB", {8, 16, 32, 64, 128, 256});
|
||||
|
||||
NVBENCH_BENCH_TYPES(hyperloglog_add, NVBENCH_TYPE_AXES(bench::defaults::key_type_range))
|
||||
.set_name("hyperloglog_add")
|
||||
.set_type_axes_names({"Key"})
|
||||
.add_int64_power_of_two_axis("NumInputs", {30})
|
||||
.add_int64_axis("SketchSizeKB", {8, 16, 32, 64, 128, 256});
|
||||
Reference in New Issue
Block a user